Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Started by Shubham Khanna12 months ago114 messages
#1Shubham Khanna
khannashubham1197@gmail.com
1 attachment(s)

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1]/messages/by-id/CAA4eK1+JpALAokLqxVsQKgo9iFrO-zChfvNXXJMkC8jUgYykBw@mail.gmail.com that was planned for
future implementation. The attached patch has the changes for the
same.

[1]: /messages/by-id/CAA4eK1+JpALAokLqxVsQKgo9iFrO-zChfvNXXJMkC8jUgYykBw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

Attachments:

v1-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v1-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 3240552a0e8853cb69191da760cdb34069d83da5 Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v1] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch introduces the ability for 'pg_createsubscriber' to automatically
fetch all non-template databases from the publisher and append them to the
subscription list.
This improvement simplifies the creation of subscriptions for multiple
databases by eliminating the need to specify each database manually.

A new function 'fetch_and_append_all_databases' in 'pg_createsubscriber.c'
that queries the publisher for all databases and adds them to the subscription
options is added.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  7 ++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 60 +++++++++++++++++++
 .../t/040_pg_createsubscriber.pl              | 52 ++++++++++++++++
 3 files changed, 118 insertions(+), 1 deletion(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..0f7c441381 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -94,7 +94,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. If this option is not provided,
+       <application>pg_createsubscriber</application> will automatically fetch
+       all non-template databases from the publisher and create subscriptions
+       for them. This is particularly useful in scenarios where the user wants
+       to convert a physical standby to a logical subscriber for
+       multiple databases, such as during an upgrade.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..db9031c377 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -106,6 +106,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_and_append_all_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -1860,6 +1861,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_and_append_all_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+	const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = PQconnectdb(opt->pub_conninfo_str);
+	/* Check for connection errors */
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		pg_log_error("Connection to the PostgreSQL server failed: %s", PQerrorMessage(conn));
+		PQfinish(conn);
+		exit(1);
+	}
+
+	res = PQexec(conn, query);
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("Failed to execute query on the PostgreSQL server: %s", PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+		num_dbs++;
+	}
+
+	/* Check if any databases were added */
+	if (opt->database_names.head == NULL)
+	{
+		pg_log_error("no database names could be fetched or specified");
+		pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option.");
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+	else
+	{
+		pg_log_info("databases were fetched and added to the subscription list");
+	}
+
+	PQclear(res);
+	PQfinish(conn);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -2115,6 +2174,7 @@ main(int argc, char **argv)
 							  progname);
 			exit(1);
 		}
+		fetch_and_append_all_databases(&opt);
 	}
 
 	/* Number of object names must match number of databases */
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..eed4b18742 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -368,6 +368,9 @@ command_ok(
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
 		'--replication-slot' => 'replslot1',
+		'--replication-slot' => 'replslot2',
+		'--replication-slot' => 'replslot3',
+		'--replication-slot' => 'replslot4',
 	],
 	'run pg_createsubscriber without --databases');
 
@@ -448,10 +451,59 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# Set up node A as primary
+my $node_a = PostgreSQL::Test::Cluster->new('node_a');
+my $aconnstr = $node_a->connstr;
+$node_a->init(allows_streaming => 'logical');
+$node_a->append_conf('postgresql.conf', 'autovacuum = off');
+$node_a->start;
+
+# Set up node B as standby linking to node A
+$node_a->backup('backup_3');
+my $node_b = PostgreSQL::Test::Cluster->new('node_b');
+$node_b->init_from_backup($node_a, 'backup_3', has_streaming => 1);
+$node_b->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$aconnstr dbname=postgres'
+hot_standby_feedback = on
+max_logical_replication_workers = 5
+]);
+$node_b->set_standby_mode();
+$node_b->start;
+# Ensure there are some user databases on the publisher
+$node_a->safe_psql('postgres', 'CREATE DATABASE db3');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db4');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db5');
+
+$node_b->stop;
+
+# pg_createsubscriber can run without --databases option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+	],
+	'run pg_createsubscriber without --databases');
+
+$node_b->start;
+# Verify subscriptions are created for all databases
+my $subscription_count =
+  $node_b->safe_psql('postgres', 'SELECT COUNT (*) FROM pg_subscription');
+is($subscription_count, '4',
+	'Three subscriptions are created for db3, db4, db5');
+
+$node_b->stop;
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
 $node_f->teardown_node;
+$node_a->teardown_node;
+$node_b->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

#2Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#1)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Jan 22, 2025 at 7:29 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1] that was planned for
future implementation. The attached patch has the changes for the
same.

I think the feature will be useful, but UI might cause some unwanted
results. If a user forgets to specify -d option, the utility will
create subscriptions to all the databases, some of which may or may
not have the publications. I think it's better to provide an option to
specify all databases explicitly (e.g. --all-databases).

--
Best Wishes,
Ashutosh Bapat

#3Peter Smith
smithpb2250@gmail.com
In reply to: Ashutosh Bapat (#2)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Jan 23, 2025 at 10:33 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Wed, Jan 22, 2025 at 7:29 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1] that was planned for
future implementation. The attached patch has the changes for the
same.

I think the feature will be useful, but UI might cause some unwanted
results. If a user forgets to specify -d option, the utility will
create subscriptions to all the databases, some of which may or may
not have the publications. I think it's better to provide an option to
specify all databases explicitly (e.g. --all-databases).

+1 better to be safe.

Instead of a new switch, how about changing the --database switch to
accept a pattern (like pg_dump --schema does [1]https://www.postgresql.org/docs/current/app-pgdump.html)

Then "all databases" would be specified something like --database = *

======
[1]: https://www.postgresql.org/docs/current/app-pgdump.html

Kind Regards,
Peter Smith.
Fujitsu Australia

#4Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Peter Smith (#3)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Jan 24, 2025 at 8:14 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Thu, Jan 23, 2025 at 10:33 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Wed, Jan 22, 2025 at 7:29 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1] that was planned for
future implementation. The attached patch has the changes for the
same.

I think the feature will be useful, but UI might cause some unwanted
results. If a user forgets to specify -d option, the utility will
create subscriptions to all the databases, some of which may or may
not have the publications. I think it's better to provide an option to
specify all databases explicitly (e.g. --all-databases).

+1 better to be safe.

Instead of a new switch, how about changing the --database switch to
accept a pattern (like pg_dump --schema does [1])

Then "all databases" would be specified something like --database = *

WFM but that will be more work than what's in the patch.

--
Best Wishes,
Ashutosh Bapat

#5Peter Smith
smithpb2250@gmail.com
In reply to: Ashutosh Bapat (#4)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Jan 24, 2025 at 7:28 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Fri, Jan 24, 2025 at 8:14 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Thu, Jan 23, 2025 at 10:33 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Wed, Jan 22, 2025 at 7:29 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1] that was planned for
future implementation. The attached patch has the changes for the
same.

I think the feature will be useful, but UI might cause some unwanted
results. If a user forgets to specify -d option, the utility will
create subscriptions to all the databases, some of which may or may
not have the publications. I think it's better to provide an option to
specify all databases explicitly (e.g. --all-databases).

+1 better to be safe.

Instead of a new switch, how about changing the --database switch to
accept a pattern (like pg_dump --schema does [1])

Then "all databases" would be specified something like --database = *

WFM but that will be more work than what's in the patch.

OK, what if, instead of full pattern matching it could recognise just
one special dbname value of '*' (meaning "all")

So, "all databases" could still be specified as --database = *

The implementation would be almost no more work than the current
patch, while at the same time leaving it open to be extended as a
pattern if needed in the future. Or, is it too hacky?

======
Kind Regards,
Peter Smith.
Fujitsu Australia

#6Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Peter Smith (#5)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Jan 28, 2025 at 3:58 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Fri, Jan 24, 2025 at 7:28 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Fri, Jan 24, 2025 at 8:14 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Thu, Jan 23, 2025 at 10:33 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Wed, Jan 22, 2025 at 7:29 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1] that was planned for
future implementation. The attached patch has the changes for the
same.

I think the feature will be useful, but UI might cause some unwanted
results. If a user forgets to specify -d option, the utility will
create subscriptions to all the databases, some of which may or may
not have the publications. I think it's better to provide an option to
specify all databases explicitly (e.g. --all-databases).

+1 better to be safe.

Instead of a new switch, how about changing the --database switch to
accept a pattern (like pg_dump --schema does [1])

Then "all databases" would be specified something like --database = *

WFM but that will be more work than what's in the patch.

OK, what if, instead of full pattern matching it could recognise just
one special dbname value of '*' (meaning "all")

So, "all databases" could still be specified as --database = *

The implementation would be almost no more work than the current
patch, while at the same time leaving it open to be extended as a
pattern if needed in the future. Or, is it too hacky?

I don't remember any precedence here. pg_dump has pg_dumpall which
dumps all the databases, so they chose to create a separate binary. If
we go that route, I think we will be able to produce a more flexible
utility, like replication slot names per database, or per database
subscription settings etc. Maybe we should consider that option.

If we want to stick to --database= supporting a pattern looks better
than just a single special pattern *.

--
Best Wishes,
Ashutosh Bapat

#7Amit Kapila
amit.kapila16@gmail.com
In reply to: Ashutosh Bapat (#6)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Jan 28, 2025 at 12:01 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

If we want to stick to --database= supporting a pattern looks better
than just a single special pattern *.

This sounds reasonable to me as well. Note that the interaction of
other parameters like --replication-slot is not yet discussed. I think
if the number of slots given matches with the number of databases
fetched based on pattern matches then we can use them otherwise,
return the ERROR. The other option could be that we don't allow
options like --replication-slot along with pattern matching option.

--
With Regards,
Amit Kapila.

#8Peter Smith
smithpb2250@gmail.com
In reply to: Amit Kapila (#7)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Jan 29, 2025 at 4:44 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Tue, Jan 28, 2025 at 12:01 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

If we want to stick to --database= supporting a pattern looks better
than just a single special pattern *.

This sounds reasonable to me as well. Note that the interaction of
other parameters like --replication-slot is not yet discussed. I think
if the number of slots given matches with the number of databases
fetched based on pattern matches then we can use them otherwise,
return the ERROR. The other option could be that we don't allow
options like --replication-slot along with pattern matching option.

I have had second thoughts about my pattern idea. Now, I favour just
adding another --all-databases switch like Ashutosh had suggested [1]/messages/by-id/CAExHW5sQGie7bvS-q7YUYDM2BqYZ=+xqeqFUS=cZGjK_9pnVzQ@mail.gmail.com
in the first place.

I had overlooked the rules saying that the user is allowed to specify
*multiple* --publication or --replication-slot or --subscription name
switches, but when doing so they have to match the same number of
--database switches. Using a --dbname=pattern would be fraught with
complications. e.g. How can we know up-front how many databases the
dbname pattern will resolve to, and even in what order they get
resolved?

In hindsight, it would be much easier to just have one extra switch,
so the rules then are simple:

--all-databases (here you CANNOT specify multiple
publication/replication-slot/subscription switches)
-database=dbname (here you CAN specify multiple
publication/replication-slot/subscription switches using the same
order and the same number as databases)

Also, --all-databases and --database switches cannot be specified at
the same time.

======
[1]: /messages/by-id/CAExHW5sQGie7bvS-q7YUYDM2BqYZ=+xqeqFUS=cZGjK_9pnVzQ@mail.gmail.com

Kind Regards,
Peter Smith.
Fujitsu Australia

#9Amit Kapila
amit.kapila16@gmail.com
In reply to: Peter Smith (#8)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Jan 30, 2025 at 6:21 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Wed, Jan 29, 2025 at 4:44 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Tue, Jan 28, 2025 at 12:01 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

If we want to stick to --database= supporting a pattern looks better
than just a single special pattern *.

This sounds reasonable to me as well. Note that the interaction of
other parameters like --replication-slot is not yet discussed. I think
if the number of slots given matches with the number of databases
fetched based on pattern matches then we can use them otherwise,
return the ERROR. The other option could be that we don't allow
options like --replication-slot along with pattern matching option.

I have had second thoughts about my pattern idea. Now, I favour just
adding another --all-databases switch like Ashutosh had suggested [1]
in the first place.

I had overlooked the rules saying that the user is allowed to specify
*multiple* --publication or --replication-slot or --subscription name
switches, but when doing so they have to match the same number of
--database switches. Using a --dbname=pattern would be fraught with
complications. e.g. How can we know up-front how many databases the
dbname pattern will resolve to, and even in what order they get
resolved?

It could be a bit tricky to find that for users but they can devise a
query to get the names and numbers of databases matching the given
pattern. OTOH, I am not sure there is a clear need at this stage for
pattern matching for this tool. So, we can go with a simple switch as
you are proposing at this stage.

--
With Regards,
Amit Kapila.

#10Shubham Khanna
khannashubham1197@gmail.com
In reply to: Amit Kapila (#9)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Jan 30, 2025 at 12:08 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Thu, Jan 30, 2025 at 6:21 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Wed, Jan 29, 2025 at 4:44 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Tue, Jan 28, 2025 at 12:01 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

If we want to stick to --database= supporting a pattern looks better
than just a single special pattern *.

This sounds reasonable to me as well. Note that the interaction of
other parameters like --replication-slot is not yet discussed. I think
if the number of slots given matches with the number of databases
fetched based on pattern matches then we can use them otherwise,
return the ERROR. The other option could be that we don't allow
options like --replication-slot along with pattern matching option.

I have had second thoughts about my pattern idea. Now, I favour just
adding another --all-databases switch like Ashutosh had suggested [1]
in the first place.

I had overlooked the rules saying that the user is allowed to specify
*multiple* --publication or --replication-slot or --subscription name
switches, but when doing so they have to match the same number of
--database switches. Using a --dbname=pattern would be fraught with
complications. e.g. How can we know up-front how many databases the
dbname pattern will resolve to, and even in what order they get
resolved?

It could be a bit tricky to find that for users but they can devise a
query to get the names and numbers of databases matching the given
pattern. OTOH, I am not sure there is a clear need at this stage for
pattern matching for this tool. So, we can go with a simple switch as
you are proposing at this stage.

After reconsidering the idea of supporting '--all-databases' switch is
the better approach at this stage, I have added the new switch in the
latest patch.
The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v2-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v2-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 5180545cbcefeb98ddc80ee22441d51253328986 Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v2] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility to automatically
fetch all non-template databases from the publisher and create subscriptions
for them. This simplifies converting a physical standby to a logical subscriber
for multiple databases, particularly during upgrades.

A new function 'fetch_all_databases' in 'pg_createsubscriber.c'
that queries the publisher for all databases and adds them to the subscription
options is added.
Additionally, 'validate_databases' ensures that conflicting options are not
used together, enforcing correct usage of '--all-databases'.

The '--all-databases' option fetches all databases from the publisher.
It auto-generates publication and replication slot names as 'pub_names' and
'slot_names' respectively.
This option validates database name lengths to ensure generated names fit
within system limits.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  18 ++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 110 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  90 ++++++++++++++
 3 files changed, 216 insertions(+), 2 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..8dea53c955 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,18 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the publisher and
+       create subscriptions for them.
+       replica.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +106,11 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. If neither <option>-d</option> nor <option>-a</option> is
+       specified, <application>pg_createsubscriber</application> will use
+       <option>--all-databases</option> by default. When using
+       <option>-a</option>, ensure database names are shorter than 59 characters
+       to allow for generated publication and slot names.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..ed06388966 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* fetch and specify all databases */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,8 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_all_databases(struct CreateSubscriberOptions *opt);
+static void validate_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +223,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             fetch and specify all databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1860,11 +1864,104 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_all_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+	const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = PQconnectdb(opt->pub_conninfo_str);
+	/* Check for connection errors */
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		pg_log_error("connection to the PostgreSQL server failed: %s", PQerrorMessage(conn));
+		PQfinish(conn);
+		exit(1);
+	}
+
+	res = PQexec(conn, query);
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("failed to execute query on the PostgreSQL server: %s", PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+		num_dbs++;
+	}
+
+	/* Check if any databases were added */
+	if (opt->database_names.head == NULL)
+	{
+		pg_log_error("no database names could be fetched or specified");
+		pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option.");
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	PQclear(res);
+	PQfinish(conn);
+}
+
+/* Function to validate all the databases and generate publication/slot names
+ * when using '--all-databases'.
+ */
+static void
+validate_databases(struct CreateSubscriberOptions *opt)
+{
+	/* Check for conflicting options */
+	if (opt->all_databases && opt->database_names.head != NULL)
+	{
+		pg_log_error("cannot specify both --all-databases and -d/--database");
+		exit(1);
+	}
+
+	/* Auto-generate parameters if using --all-databases */
+	if (opt->all_databases)
+	{
+		/* Generate publication and slot names if not specified */
+		SimpleStringListCell *cell;
+
+		fetch_all_databases(opt);
+
+		cell = opt->database_names.head;
+
+		while (cell != NULL)
+		{
+			char		slot_name[NAMEDATALEN];
+
+			snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+			simple_string_list_append(&opt->replslot_names, slot_name);
+
+			snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+			simple_string_list_append(&opt->pub_names, slot_name);
+
+			cell = cell->next;
+		}
+	}
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1927,6 +2024,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1949,11 +2047,15 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_databases = true;
+				break;
+
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2091,6 +2193,10 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/* Validate and process database options */
+	if (opt.all_databases)
+		validate_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
@@ -2117,6 +2223,8 @@ main(int argc, char **argv)
 		}
 	}
 
+
+
 	/* Number of object names must match number of databases */
 	if (num_pubs > 0 && num_pubs != num_dbs)
 	{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..90874ae909 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -448,10 +448,100 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# Set up node A as primary
+my $node_a = PostgreSQL::Test::Cluster->new('node_a');
+my $aconnstr = $node_a->connstr;
+$node_a->init(allows_streaming => 'logical');
+$node_a->append_conf('postgresql.conf', 'autovacuum = off');
+$node_a->start;
+
+# Set up node B as standby linking to node A
+$node_a->backup('backup_3');
+my $node_b = PostgreSQL::Test::Cluster->new('node_b');
+$node_b->init_from_backup($node_a, 'backup_3', has_streaming => 1);
+$node_b->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$aconnstr dbname=postgres'
+hot_standby_feedback = on
+max_logical_replication_workers = 5
+]);
+$node_b->set_standby_mode();
+$node_b->start;
+
+# Fetch the count of non-template databases on the publisher before
+# running pg_createsubscriber without --all-databases option
+my $db_count_before =
+  $node_a->safe_psql('postgres',
+	"SELECT count(*) FROM pg_database WHERE datistemplate = false;");
+is($db_count_before, '1', 'database count without --all-databases option');
+
+# Ensure there are some user databases on the publisher
+$node_a->safe_psql('postgres', 'CREATE DATABASE db3');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db4');
+
+$node_b->stop;
+
+# run pg_createsubscriber publication and slot and verify the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+		'--publication' => 'pub1',
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/cannot use --all-databases with --publication or --replication-slot/,
+	'fail if --all-databases is used with publication and slot');
+
+# run pg_createsubscriber '--dbname' and '--all-databases' and verify the
+# conflict issue
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+		'--dbname' => 'db3',
+		'--all-databases',
+	],
+	qr/cannot use --dbname with --all-databases/,
+	'fail if --dbname is used with --all-databases');
+
+# run pg_createsubscriber with --all-databases option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
+$node_b->start;
+# Fetch the count of non-template databases on the subscriber after
+# running pg_createsubscriber with --all-databases option
+my $db_count_after =
+  $node_b->safe_psql('postgres',
+	"SELECT count(*) FROM pg_database WHERE datistemplate = false;");
+is($db_count_after, '3', 'database count with --all-databases option');
+
+$node_b->stop;
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
 $node_f->teardown_node;
+$node_a->teardown_node;
+$node_b->teardown_node;
 
 done_testing();
-- 
2.34.1

#11Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#10)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi Shubham,

On Tue, Feb 4, 2025 at 2:10 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

It could be a bit tricky to find that for users but they can devise a
query to get the names and numbers of databases matching the given
pattern. OTOH, I am not sure there is a clear need at this stage for
pattern matching for this tool. So, we can go with a simple switch as
you are proposing at this stage.

After reconsidering the idea of supporting '--all-databases' switch is
the better approach at this stage, I have added the new switch in the
latest patch.
The attached patch contains the suggested changes.

+ If neither <option>-d</option> nor <option>-a</option> is
+       specified, <application>pg_createsubscriber</application> will use
+       <option>--all-databases</option> by default.

As pointed upthread by me and Peter, using --all-databases by default
is not a good behaviour.

But the code doesn't behave like --all-databases by default. Looks
like we need to fix the documentation.

+ /* Generate publication and slot names if not specified */
+ SimpleStringListCell *cell;
+
+ fetch_all_databases(opt);
+
+ cell = opt->database_names.head;

We don't seem to check existence of publication and slot name
specification as the comment indicates. Do we need to check that those
names are not specified at all? and also mention in the documentation
that those specifications are required when using -a/--all-databases
option?

--
Best Wishes,
Ashutosh Bapat

#12Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#11)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Feb 4, 2025 at 3:49 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

Hi Shubham,

On Tue, Feb 4, 2025 at 2:10 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

It could be a bit tricky to find that for users but they can devise a
query to get the names and numbers of databases matching the given
pattern. OTOH, I am not sure there is a clear need at this stage for
pattern matching for this tool. So, we can go with a simple switch as
you are proposing at this stage.

After reconsidering the idea of supporting '--all-databases' switch is
the better approach at this stage, I have added the new switch in the
latest patch.
The attached patch contains the suggested changes.

+ If neither <option>-d</option> nor <option>-a</option> is
+       specified, <application>pg_createsubscriber</application> will use
+       <option>--all-databases</option> by default.

As pointed upthread by me and Peter, using --all-databases by default
is not a good behaviour.

But the code doesn't behave like --all-databases by default. Looks
like we need to fix the documentation.

Updated the documentation accordingly and added the current behaviour
of -a/--all-databases option.

+ /* Generate publication and slot names if not specified */
+ SimpleStringListCell *cell;
+
+ fetch_all_databases(opt);
+
+ cell = opt->database_names.head;

We don't seem to check existence of publication and slot name
specification as the comment indicates. Do we need to check that those
names are not specified at all? and also mention in the documentation
that those specifications are required when using -a/--all-databases
option?

Added a check to verify that publication and slot names are not
manually specified when using -a/--all-databases option and updated
the documentation accordingly.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v3-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v3-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 3bff40f826f11b72ea4df1c6166bb62231ec46db Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v3] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility to automatically
fetch all non-template databases from the publisher and create subscriptions
for them. This simplifies converting a physical standby to a logical subscriber
for multiple databases, particularly during upgrades.

A new function 'fetch_all_databases' in 'pg_createsubscriber.c'
that queries the publisher for all databases and adds them to the subscription
options is added.
Additionally, 'validate_databases' ensures that conflicting options are not
used together, enforcing correct usage of '--all-databases'.

The '--all-databases' option fetches all databases from the publisher.
It auto-generates publication and replication slot names as 'pub_names' and
'slot_names' respectively.
This option validates database name lengths to ensure generated names fit
within system limits.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  22 +++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 120 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  91 +++++++++++++
 3 files changed, 231 insertions(+), 2 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..5dc05736d3 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,24 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the publisher and
+       create subscriptions for them.
+       When using <option>--all-databases</option>, publication and
+       replication slot names will be automatically generated in the format
+       <literal><replaceable>database</replaceable>_pub</literal> and
+       <literal><replaceable>database</replaceable>_slot</literal> respectively.
+       When using <option>-a</option>, ensure database names are shorter
+       than 59 characters to allow for generated publication and slot names.
+       replica.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +112,9 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. If <option>-d</option> and <option>-a</option> are specified
+       together, <application>pg_createsubscriber</application> will return
+       an error.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..7e869ee7cc 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* fetch and specify all databases */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,8 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_all_databases(struct CreateSubscriberOptions *opt);
+static void validate_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +223,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             fetch and specify all databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1860,11 +1864,114 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_all_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+	const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = PQconnectdb(opt->pub_conninfo_str);
+	/* Check for connection errors */
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		pg_log_error("connection to the PostgreSQL server failed: %s", PQerrorMessage(conn));
+		PQfinish(conn);
+		exit(1);
+	}
+
+	res = PQexec(conn, query);
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("failed to execute query on the PostgreSQL server: %s", PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+		num_dbs++;
+	}
+
+	/* Check if any databases were added */
+	if (opt->database_names.head == NULL)
+	{
+		pg_log_error("no database names could be fetched or specified");
+		pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option.");
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	PQclear(res);
+	PQfinish(conn);
+}
+
+/* Function to validate all the databases and generate publication/slot names
+ * when using '--all-databases'.
+ */
+static void
+validate_databases(struct CreateSubscriberOptions *opt)
+{
+	/* Check for conflicting options */
+	if (opt->all_databases && opt->database_names.head != NULL)
+	{
+		pg_log_error("cannot specify --dbname when using --all-databases");
+		exit(1);
+	}
+
+	/*
+	 * Ensure publication and slot names are not manually specified with
+	 * --all-databases
+	 */
+	if (opt->all_databases &&
+		(opt->pub_names.head != NULL || opt->replslot_names.head != NULL))
+	{
+		pg_log_error("cannot specify --publication or --replication-slot when using --all-databases");
+		exit(1);
+	}
+
+	/* Auto-generate publication and slot names for all databases */
+	if (opt->all_databases)
+	{
+		SimpleStringListCell *cell;
+
+		fetch_all_databases(opt);
+
+		cell = opt->database_names.head;
+
+		while (cell != NULL)
+		{
+			char		slot_name[NAMEDATALEN];
+
+			snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+			simple_string_list_append(&opt->replslot_names, slot_name);
+
+			snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+			simple_string_list_append(&opt->pub_names, slot_name);
+
+			cell = cell->next;
+		}
+	}
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1927,6 +2034,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1949,11 +2057,15 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_databases = true;
+				break;
+
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2091,6 +2203,10 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/* Validate and process database options */
+	if (opt.all_databases)
+		validate_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
@@ -2117,6 +2233,8 @@ main(int argc, char **argv)
 		}
 	}
 
+
+
 	/* Number of object names must match number of databases */
 	if (num_pubs > 0 && num_pubs != num_dbs)
 	{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..45ed22b97d 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -448,10 +448,101 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# Set up node A as primary
+my $node_a = PostgreSQL::Test::Cluster->new('node_a');
+my $aconnstr = $node_a->connstr;
+$node_a->init(allows_streaming => 'logical');
+$node_a->append_conf('postgresql.conf', 'autovacuum = off');
+$node_a->start;
+
+# Set up node B as standby linking to node A
+$node_a->backup('backup_3');
+my $node_b = PostgreSQL::Test::Cluster->new('node_b');
+$node_b->init_from_backup($node_a, 'backup_3', has_streaming => 1);
+$node_b->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$aconnstr dbname=postgres'
+hot_standby_feedback = on
+max_logical_replication_workers = 5
+]);
+$node_b->set_standby_mode();
+$node_b->start;
+
+# Fetch the count of non-template databases on the publisher before
+# running pg_createsubscriber without '--all-databases' option
+my $db_count_before =
+  $node_a->safe_psql('postgres',
+	"SELECT count(*) FROM pg_database WHERE datistemplate = false;");
+is($db_count_before, '1', 'database count without --all-databases option');
+
+# Ensure there are some user databases on the publisher
+$node_a->safe_psql('postgres', 'CREATE DATABASE db3');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db4');
+
+$node_b->stop;
+
+# run pg_createsubscriber with '--publication' and '--replication-slot' and
+# verify the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+		'--publication' => 'pub1',
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/cannot specify --publication or --replication-slot when using --all-databases/,
+	'fail if --all-databases is used with publication and slot');
+
+# run pg_createsubscriber with '--dbname' and '--all-databases' and verify the
+# failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+		'--dbname' => 'db3',
+		'--all-databases',
+	],
+	qr/cannot specify --dbname when using --all-databases/,
+	'fail if --dbname is used with --all-databases');
+
+# run pg_createsubscriber with '--all-databases' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
+$node_b->start;
+# Fetch the count of non-template databases on the subscriber after
+# running pg_createsubscriber with '--all-databases' option
+my $db_count_after =
+  $node_b->safe_psql('postgres',
+	"SELECT count(*) FROM pg_database WHERE datistemplate = false;");
+is($db_count_after, '3', 'database count with --all-databases option');
+
+$node_b->stop;
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
 $node_f->teardown_node;
+$node_a->teardown_node;
+$node_b->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

#13Shlok Kyal
shlok.kyal.oss@gmail.com
In reply to: Shubham Khanna (#12)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, 5 Feb 2025 at 01:26, Shubham Khanna <khannashubham1197@gmail.com> wrote:

On Tue, Feb 4, 2025 at 3:49 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

Hi Shubham,

On Tue, Feb 4, 2025 at 2:10 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

It could be a bit tricky to find that for users but they can devise a
query to get the names and numbers of databases matching the given
pattern. OTOH, I am not sure there is a clear need at this stage for
pattern matching for this tool. So, we can go with a simple switch as
you are proposing at this stage.

After reconsidering the idea of supporting '--all-databases' switch is
the better approach at this stage, I have added the new switch in the
latest patch.
The attached patch contains the suggested changes.

+ If neither <option>-d</option> nor <option>-a</option> is
+       specified, <application>pg_createsubscriber</application> will use
+       <option>--all-databases</option> by default.

As pointed upthread by me and Peter, using --all-databases by default
is not a good behaviour.

But the code doesn't behave like --all-databases by default. Looks
like we need to fix the documentation.

Updated the documentation accordingly and added the current behaviour
of -a/--all-databases option.

+ /* Generate publication and slot names if not specified */
+ SimpleStringListCell *cell;
+
+ fetch_all_databases(opt);
+
+ cell = opt->database_names.head;

We don't seem to check existence of publication and slot name
specification as the comment indicates. Do we need to check that those
names are not specified at all? and also mention in the documentation
that those specifications are required when using -a/--all-databases
option?

Added a check to verify that publication and slot names are not
manually specified when using -a/--all-databases option and updated
the documentation accordingly.

The attached patch contains the suggested changes.

Hi Shubham,

Here are some of my comments:

1. We should start the comment text from the next line
+
+/* Function to validate all the databases and generate publication/slot names
+ * when using '--all-databases'.
+ */
2. Here in error message it should be '--database'
+ /* Check for conflicting options */
+ if (opt->all_databases && opt->database_names.head != NULL)
+ {
+ pg_log_error("cannot specify --dbname when using --all-databases");
+ exit(1);
+ }
+
3. I think checking 'opt->all_databases' in if conditions in function
'validate_databases' is not required
+static void
+validate_databases(struct CreateSubscriberOptions *opt)
+{
+ /* Check for conflicting options */
+ if (opt->all_databases && opt->database_names.head != NULL)
+ {
+ pg_log_error("cannot specify --dbname when using --all-databases");
+ exit(1);
+ }
as we are already checking it while calling the function
+ /* Validate and process database options */
+ if (opt.all_databases)
+ validate_databases(&opt);
+

4. Here we should update the count of 'num_replslots' and 'num_pubs'

+ while (cell != NULL)
+ {
+ char slot_name[NAMEDATALEN];
+
+ snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+ simple_string_list_append(&opt->replslot_names, slot_name);
+
+ snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+ simple_string_list_append(&opt->pub_names, slot_name);
+
+ cell = cell->next;
+ }
Since we are not updating these counts, the names are reflected as expected.

Should also store subscription names similarly? Or maybe store
subscription names and assign slot names same as subscription names?

5. Since --all-databases option is added we should update the comment:
/*
* If --database option is not provided, try to obtain the dbname from
* the publisher conninfo. If dbname parameter is not available, error
* out.
*/

6. Extra blank lines should be removed
}
}

+
+
  /* Number of object names must match number of databases */

Thanks and Regards,
Shlok Kyal

#14Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#12)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for working on it. I have some comments for the patch.

01. fetch_all_databases
```
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_all_databases(struct CreateSubscriberOptions *opt)
```

Please update the comment atop function.

02. fetch_all_databases
```
+	/* Establish a connection to the PostgreSQL server */
+	conn = PQconnectdb(opt->pub_conninfo_str);
+	/* Check for connection errors */
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		pg_log_error("connection to the PostgreSQL server failed: %s", PQerrorMessage(conn));
+		PQfinish(conn);
+		exit(1);
+	}
```

You can use connect_database() instead of directly calling PQconnectdb().

03. fetch_all_databases
```
+ const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
```

We should verify the attribute datallowconn, which indicates we can connect to the
database. If it is false, no one would be able to connect to the db and define anything.

04. fetch_all_databases
```
+	/* Check if any databases were added */
+	if (opt->database_names.head == NULL)
+	{
+		pg_log_error("no database names could be fetched or specified");
+		pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option.");
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
```

It is enough to check num_rows > 0.

05. fetch_all_databases
```
+ PQfinish(conn);
```

Just in case: we can use disconnect_database().

06. validate_databases
```
+	/* Check for conflicting options */
+	if (opt->all_databases && opt->database_names.head != NULL)
+	{
+		pg_log_error("cannot specify --dbname when using --all-databases");
+		exit(1);
+	}
+
+	/*
+	 * Ensure publication and slot names are not manually specified with
+	 * --all-databases
+	 */
+	if (opt->all_databases &&
+		(opt->pub_names.head != NULL || opt->replslot_names.head != NULL))
+	{
+		pg_log_error("cannot specify --publication or --replication-slot when using --all-databases");
+		exit(1);
+	}
```

I think validations should be done in main() like other paramters.

07. validate_databases
```
+ if (opt->all_databases)
+ {
```

This function is called when all_databases is true, so this is not needed.

08. validate_databases
```
+		cell = opt->database_names.head;
+
+		while (cell != NULL)
+		{
+			char		slot_name[NAMEDATALEN];
+
+			snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+			simple_string_list_append(&opt->replslot_names, slot_name);
+
+			snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+			simple_string_list_append(&opt->pub_names, slot_name);
+
+			cell = cell->next;
+		}
```

I'm not sure the part. It seems to me that you tried to set slotname to ${dbname}_slot
and pubname to ${dbname}_pub, but this the current naming rule. Since we can
generate names in setup_publisher(), I think we do not have to do something here.

09.
```
@@ -2117,6 +2233,8 @@ main(int argc, char **argv)
}
}

+
+
```

Unnesesarry blank.

10. 040_pg_createsubscriber.pl
```
+# Fetch the count of non-template databases on the publisher before
+# running pg_createsubscriber without '--all-databases' option
+my $db_count_before =
+  $node_a->safe_psql('postgres',
+	"SELECT count(*) FROM pg_database WHERE datistemplate = false;");
+is($db_count_before, '1', 'database count without --all-databases option');
```

This test is not actually realted with our codes, no need to do.

11. 040_pg_createsubscriber.pl
```
+# Ensure there are some user databases on the publisher
+$node_a->safe_psql('postgres', 'CREATE DATABASE db3');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db4');
+
+$node_b->stop;
```

You must wait until all changes have been replicated to the standby here.

12. 040_pg_createsubscriber.pl

Can we do the similar tests without adding new instances? E.g., runs with
dry-run mode and confirms all databases become target.

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#15Peter Smith
smithpb2250@gmail.com
In reply to: Shubham Khanna (#12)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi Shubham.

Here are some review comments for v3-0001.

FYI, I skipped the review of the test code because I saw Kuroda-san
had already posted some comments about the tests

======
Commit message

1.
This patch enhances the 'pg_createsubscriber' utility to automatically
fetch all non-template databases from the publisher and create subscriptions
for them. This simplifies converting a physical standby to a logical subscriber
for multiple databases, particularly during upgrades.

~

AFAIK you are not creating the subscription at the publisher database,
which is what this description sounds like it is saying. I think you
are creating the subscriptions on the target server on the *same name*
databases that you found on the source server (???). Anyway, this
needs clarification. Also, in some of the comments below, I had the
same confusion.

~~~

2.
A new function 'fetch_all_databases' in 'pg_createsubscriber.c'
that queries the publisher for all databases and adds them to the subscription
options is added.
Additionally, 'validate_databases' ensures that conflicting options are not
used together, enforcing correct usage of '--all-databases'.

~

IMO it is better here to describe the overall logic/algorithms rather
than saying "This new function does this, and this other new function
does that". (otherwise, your commit message will quickly become stale
and need changing all the time).

~~~

3.
The '--all-databases' option fetches all databases from the publisher.
It auto-generates publication and replication slot names as 'pub_names' and
'slot_names' respectively.

~

'pub_names'?
'slot_names'?

What are those strings? Those are not the names that you wrote in the document.

~~~

4.
This option validates database name lengths to ensure generated names fit
within system limits.

~

No. Maybe the patch code ensures this (but I didn't find this
validation anywhere), but certainly "This option" doesn't.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

5.
The synopsis has no mention of --all-databases.

I'm not sure of the best way to do it -- if it becomes too messy to
combine everything then maybe having several is the way to go:

e.g.

pg_createsubscriber [option...] { -a | --all-databases } { -D |
--pgdata }datadir { -P | --publisher-server }connstr
pg_createsubscriber [option...] { -d | --database }dbname { -D |
--pgdata }datadir { -P | --publisher-server }connstr

~~~

6.
+      <para>
+       Automatically fetch all non-template databases from the publisher and
+       create subscriptions for them.
+       When using <option>--all-databases</option>, publication and
+       replication slot names will be automatically generated in the format
+       <literal><replaceable>database</replaceable>_pub</literal> and
+       <literal><replaceable>database</replaceable>_slot</literal>
respectively.
+       When using <option>-a</option>, ensure database names are shorter
+       than 59 characters to allow for generated publication and slot names.
+       replica.
+      </para>

6a.
"fetch all non-template databases from the publisher and create
subscriptions for them" -- Hmm. It's too confusing. I guess you meant
that you are enumerating all the databases on the source server but
then creating subscriptions on all the databases at the target server
that have the same name (???) Anyway, the current text is not clear. I
also think it is better to use the "target server" and "source server"
(aka publication-server) terminology.

~

6b.
I don't think you need to say "When using
<option>--all-databases</option>," because this entire description for
'--all-databases'

~

6c.
Talks about publication and slot names but no mention of subscription names?

~

6d.
That limitation about the 59 char limit seems a strange thing to have
documented here. I wonder why you need to say anything at all. It
might be better if the code just validates all this up-front before
processing and can raise an ERROR in the very rare times this might
happen.

~

6e.
Spurious text junk? "replica"

~

6f.
Don't you need to also say something like "Cannot be used with
'--publication', '--subscription', or '--replication-slot'."

~~~

7. --database

-       switches.
+       switches. If <option>-d</option> and <option>-a</option> are specified
+       together, <application>pg_createsubscriber</application> will return
+       an error.

A nicer way to say that would be to spell it out fully. e.g. "Cannot
be used with --all-databases'.".

~~~

8.
For --publication don't you need to mention "Cannot be used with
--all-databases'.".

~~~

9.
For --subscription don't you need to mention "Cannot be used with
--all-databases'.".

~~~

10.
For --replication-slot don't you need to mention "Cannot be used with
--all-databases'.".

======
src/bin/pg_basebackup/pg_createsubscriber.c

usage:

11.
printf(_("\nOptions:\n"));
+ printf(_(" -a, --all-databases fetch and specify all
databases\n"));

This doesn't really seem a helpful usage text. Should it say something
more like "create subscriptions for all databases"... (??) **

** I have difficulty imagining what is the expected behaviour in cases
where there might be different databases at the source server and
target server. I think your documentation needs to have some special
notes about this to make it clear what server you are enumerating
these databases on, and then what happens if there is some mismatch of
what databases are available on the *other* server.

~~~

fetch_all_databases:

12.
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_all_databases(struct CreateSubscriberOptions *opt)
+{

12a.
What is a "placeholder function"?

~

12b.
I would be nicers to use the same terminology that seems more common
in the documentation -- e.g. "source server" instead of "publisher".
Also, the function name should make it clear if you are getting these
data from the source server or target server.

~~~

13.
+ /* Process the query result */
+ num_rows = PQntuples(res);
+ for (int i = 0; i < num_rows; i++)
+ {
+ const char *dbname = PQgetvalue(res, i, 0);
+
+ simple_string_list_append(&opt->database_names, dbname);
+ num_dbs++;
+ }
The increment num_dbs++ is a bit sneaky. AFAICT you are effectively
pretending --all-databases is the equivalent of a whole lot of
--database so the subsequent logic won't know the difference. I think
that logic deserves a code comment.

~~~

14.
+ /* Check if any databases were added */
+ if (opt->database_names.head == NULL)
+ {
+ pg_log_error("no database names could be fetched or specified");
+ pg_log_error_hint("Ensure that databases exist on the publisher or
specify a database using --database option.");
+ PQclear(res);
+ PQfinish(conn);
+ exit(1);
+ }
+
+ PQclear(res);
+ PQfinish(conn);
+}

14a.
The "were added" seems odd. I think you mean. Just "Error if no
databases were found".

~

14b.
Isn't it simpler just to check if num_dbs == 0?

~

14c.
"no database names could be fetched or specified" seems like a weird
error text. e.g. "specified"?

~

14d.
The hint seems strange to me. If there are no databases found using
--all-database then how will "specify a database using --database
option." help the user get a better result?

~~~

validate_databases:

15.
+static void
+validate_databases(struct CreateSubscriberOptions *opt)
+{
+ /* Check for conflicting options */
+ if (opt->all_databases && opt->database_names.head != NULL)
+ {
+ pg_log_error("cannot specify --dbname when using --all-databases");
+ exit(1);
+ }
+
+ /*
+ * Ensure publication and slot names are not manually specified with
+ * --all-databases
+ */
+ if (opt->all_databases &&
+ (opt->pub_names.head != NULL || opt->replslot_names.head != NULL))
+ {
+ pg_log_error("cannot specify --publication or --replication-slot
when using --all-databases");
+ exit(1);
+ }
+

I think all this switch-checking is misplaced. It should be happening
up-front in the main function.

~~~

16.
+ /* Auto-generate publication and slot names for all databases */
+ if (opt->all_databases)
+ {
+ SimpleStringListCell *cell;
+
+ fetch_all_databases(opt);
+
+ cell = opt->database_names.head;
+
+ while (cell != NULL)
+ {
+ char slot_name[NAMEDATALEN];
+
+ snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+ simple_string_list_append(&opt->replslot_names, slot_name);
+
+ snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+ simple_string_list_append(&opt->pub_names, slot_name);
+
+ cell = cell->next;
+ }
+ }

16a.
Why is this code checking 'opt->all_databases'? Isn't it only possible
to get to this function via --all-databases. You can just Assert this.

~

16b.
Why are you reusing 'slot_name' variable like this?

~

16c.
Was there some ERROR checking missing here to ensure the length of the
dbname is not going to cause pub/slot to exceed NAMEDATALEN?

~

16d.
In hindsight, most of this function seems unnecessary to me. Probably
you could've done all this pub/slot name assignment inside the
fetch_all_databases() at the same time as you were doing
simple_string_list_append(&opt->database_names, dbname); for each
database.

~~~

17.
- while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+ while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
  long_options, &option_index)) != -1)

Isn't the code within this loop missing all the early switch
validation for checking you are not combining incompatible switches?

e.g.
--all-databases - "Cannot specify --all-databases more than once"
--all-databases - "Cannot combine with --database, --publication,
--subscription, --replication-slot"
--database - "Cannot combine with --all-databases"
--publication - "Cannot combine with --all-databases"
--subscription - "Cannot combine with --all-databases"
--replication-slot - "Cannot combine with --all-databases"

~~~

18.
+
+
  /* Number of object names must match number of databases */

Remove spurious whitespace.

======
Kind Regards,
Peter Smith.
Fujitsu Australia

#16Shubham Khanna
khannashubham1197@gmail.com
In reply to: Shlok Kyal (#13)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 5, 2025 at 5:31 PM Shlok Kyal <shlok.kyal.oss@gmail.com> wrote:

On Wed, 5 Feb 2025 at 01:26, Shubham Khanna <khannashubham1197@gmail.com> wrote:

On Tue, Feb 4, 2025 at 3:49 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

Hi Shubham,

On Tue, Feb 4, 2025 at 2:10 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

It could be a bit tricky to find that for users but they can devise a
query to get the names and numbers of databases matching the given
pattern. OTOH, I am not sure there is a clear need at this stage for
pattern matching for this tool. So, we can go with a simple switch as
you are proposing at this stage.

After reconsidering the idea of supporting '--all-databases' switch is
the better approach at this stage, I have added the new switch in the
latest patch.
The attached patch contains the suggested changes.

+ If neither <option>-d</option> nor <option>-a</option> is
+       specified, <application>pg_createsubscriber</application> will use
+       <option>--all-databases</option> by default.

As pointed upthread by me and Peter, using --all-databases by default
is not a good behaviour.

But the code doesn't behave like --all-databases by default. Looks
like we need to fix the documentation.

Updated the documentation accordingly and added the current behaviour
of -a/--all-databases option.

+ /* Generate publication and slot names if not specified */
+ SimpleStringListCell *cell;
+
+ fetch_all_databases(opt);
+
+ cell = opt->database_names.head;

We don't seem to check existence of publication and slot name
specification as the comment indicates. Do we need to check that those
names are not specified at all? and also mention in the documentation
that those specifications are required when using -a/--all-databases
option?

Added a check to verify that publication and slot names are not
manually specified when using -a/--all-databases option and updated
the documentation accordingly.

The attached patch contains the suggested changes.

Hi Shubham,

Here are some of my comments:

1. We should start the comment text from the next line
+
+/* Function to validate all the databases and generate publication/slot names
+ * when using '--all-databases'.
+ */
2. Here in error message it should be '--database'
+ /* Check for conflicting options */
+ if (opt->all_databases && opt->database_names.head != NULL)
+ {
+ pg_log_error("cannot specify --dbname when using --all-databases");
+ exit(1);
+ }
+
3. I think checking 'opt->all_databases' in if conditions in function
'validate_databases' is not required
+static void
+validate_databases(struct CreateSubscriberOptions *opt)
+{
+ /* Check for conflicting options */
+ if (opt->all_databases && opt->database_names.head != NULL)
+ {
+ pg_log_error("cannot specify --dbname when using --all-databases");
+ exit(1);
+ }
as we are already checking it while calling the function
+ /* Validate and process database options */
+ if (opt.all_databases)
+ validate_databases(&opt);
+

4. Here we should update the count of 'num_replslots' and 'num_pubs'

+ while (cell != NULL)
+ {
+ char slot_name[NAMEDATALEN];
+
+ snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+ simple_string_list_append(&opt->replslot_names, slot_name);
+
+ snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+ simple_string_list_append(&opt->pub_names, slot_name);
+
+ cell = cell->next;
+ }
Since we are not updating these counts, the names are reflected as expected.

Should also store subscription names similarly? Or maybe store
subscription names and assign slot names same as subscription names?

5. Since --all-databases option is added we should update the comment:
/*
* If --database option is not provided, try to obtain the dbname from
* the publisher conninfo. If dbname parameter is not available, error
* out.
*/

6. Extra blank lines should be removed
}
}

+
+
/* Number of object names must match number of databases */

Fixed the given comments. The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v4-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v4-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 1785f9ab45dd2ce111eb67ee2a681e493f3c6942 Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v4] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option, which automatically fetches all non-template
databases on the source server (publisher) and creates corresponding
subscriptions on the target server (subscriber). This simplifies the process
of converting a physical standby to a logical subscriber, particularly
during upgrades.

When '--all-databases' is used, the tool queries the source server for all
databases and attempts to create subscriptions on the target server for
databases with matching names. The tool auto-generates subscription,publication
and replication slot names using the format:
  - Subscription: '<database>_sub'
  - Publication: '<database>_pub'
  - Replication slot: '<database>_slot'

The patch ensures that conflicting options such as '--all-databases' and
'--database', '--publication', '--subscription', or '--replication-slot' cannot
be used together.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  36 ++++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 101 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  49 +++++++++
 3 files changed, 178 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..4a9dded234 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,32 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, it is
+       skipped or an error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
+       Subscription names, publication names, and replication slot names are
+       automatically generated using the format:
+       <literal><replaceable>database</replaceable>_sub</literal>
+       <literal><replaceable>database</replaceable>_pub</literal> and
+       <literal><replaceable>database</replaceable>_slot</literal> respectively.
+      </para>
+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--publication</option>, <option>--subscription</option>,
+       or <option>--replication-slot</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +120,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>-a</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +240,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>-a</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +257,7 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with <option>-a</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +272,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>-a</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..3e6dfb02fe 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* fetch and specify all databases */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             create subscriptions for all matching databases on the target\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1860,11 +1863,69 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/* Fetch all databases from the source server */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+	const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	/* Check for connection errors */
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		pg_log_error("connection to the source server failed: %s", PQerrorMessage(conn));
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	res = PQexec(conn, query);
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("failed to execute query on the source server: %s", PQerrorMessage(conn));
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_rows == 0)
+	{
+		pg_log_error("no databases found on the source server");
+		pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1927,6 +1988,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1949,12 +2011,21 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_databases = true;
+				break;
+
 			case 'd':
+				if (opt.all_databases)
+				{
+					pg_log_error("--database cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
 					simple_string_list_append(&opt.database_names, optarg);
@@ -1977,6 +2048,11 @@ main(int argc, char **argv)
 				opt.sub_port = pg_strdup(optarg);
 				break;
 			case 'P':
+				if (opt.all_databases)
+				{
+					pg_log_error("--publication cannot be used with --all-databases");
+					exit(1);
+				}
 				opt.pub_conninfo_str = pg_strdup(optarg);
 				break;
 			case 's':
@@ -2008,6 +2084,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 3:
+				if (opt.all_databases)
+				{
+					pg_log_error("--replslot cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.replslot_names, optarg))
 				{
 					simple_string_list_append(&opt.replslot_names, optarg);
@@ -2020,6 +2101,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 4:
+				if (opt.all_databases)
+				{
+					pg_log_error("--subscription cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.sub_names, optarg))
 				{
 					simple_string_list_append(&opt.sub_names, optarg);
@@ -2091,14 +2177,21 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all-databases is
+	 * specified.
+	 */
+	if (opt.all_databases)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all-databases option is provided, try
+		 * to obtain the dbname from the publisher conninfo. If dbname
+		 * parameter is not available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..af2c991ee5 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,55 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--publication' and '--replication-slot' and
+# verify the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/cannot specify --publication or --replication-slot when using --all-databases/,
+	'fail if --all-databases is used with publication and slot');
+
+# run pg_createsubscriber with '--dbname' and '--all-databases' and verify the
+# failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--dbname' => 'db1',
+		'--all-databases',
+	],
+	qr/cannot specify --dbname when using --all-databases/,
+	'fail if --dbname is used with --all-databases');
+
+# run pg_createsubscriber with '--all-databases' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
-- 
2.34.1

#17Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#14)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Feb 6, 2025 at 7:37 AM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for working on it. I have some comments for the patch.

01. fetch_all_databases
```
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_all_databases(struct CreateSubscriberOptions *opt)
```

Please update the comment atop function.

02. fetch_all_databases
```
+       /* Establish a connection to the PostgreSQL server */
+       conn = PQconnectdb(opt->pub_conninfo_str);
+       /* Check for connection errors */
+       if (PQstatus(conn) != CONNECTION_OK)
+       {
+               pg_log_error("connection to the PostgreSQL server failed: %s", PQerrorMessage(conn));
+               PQfinish(conn);
+               exit(1);
+       }
```

You can use connect_database() instead of directly calling PQconnectdb().

03. fetch_all_databases
```
+ const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
```

We should verify the attribute datallowconn, which indicates we can connect to the
database. If it is false, no one would be able to connect to the db and define anything.

04. fetch_all_databases
```
+       /* Check if any databases were added */
+       if (opt->database_names.head == NULL)
+       {
+               pg_log_error("no database names could be fetched or specified");
+               pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option.");
+               PQclear(res);
+               PQfinish(conn);
+               exit(1);
+       }
```

It is enough to check num_rows > 0.

05. fetch_all_databases
```
+ PQfinish(conn);
```

Just in case: we can use disconnect_database().

06. validate_databases
```
+       /* Check for conflicting options */
+       if (opt->all_databases && opt->database_names.head != NULL)
+       {
+               pg_log_error("cannot specify --dbname when using --all-databases");
+               exit(1);
+       }
+
+       /*
+        * Ensure publication and slot names are not manually specified with
+        * --all-databases
+        */
+       if (opt->all_databases &&
+               (opt->pub_names.head != NULL || opt->replslot_names.head != NULL))
+       {
+               pg_log_error("cannot specify --publication or --replication-slot when using --all-databases");
+               exit(1);
+       }
```

I think validations should be done in main() like other paramters.

07. validate_databases
```
+ if (opt->all_databases)
+ {
```

This function is called when all_databases is true, so this is not needed.

08. validate_databases
```
+               cell = opt->database_names.head;
+
+               while (cell != NULL)
+               {
+                       char            slot_name[NAMEDATALEN];
+
+                       snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+                       simple_string_list_append(&opt->replslot_names, slot_name);
+
+                       snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+                       simple_string_list_append(&opt->pub_names, slot_name);
+
+                       cell = cell->next;
+               }
```

I'm not sure the part. It seems to me that you tried to set slotname to ${dbname}_slot
and pubname to ${dbname}_pub, but this the current naming rule. Since we can
generate names in setup_publisher(), I think we do not have to do something here.

09.
```
@@ -2117,6 +2233,8 @@ main(int argc, char **argv)
}
}

+
+
```

Unnesesarry blank.

10. 040_pg_createsubscriber.pl
```
+# Fetch the count of non-template databases on the publisher before
+# running pg_createsubscriber without '--all-databases' option
+my $db_count_before =
+  $node_a->safe_psql('postgres',
+       "SELECT count(*) FROM pg_database WHERE datistemplate = false;");
+is($db_count_before, '1', 'database count without --all-databases option');
```

This test is not actually realted with our codes, no need to do.

11. 040_pg_createsubscriber.pl
```
+# Ensure there are some user databases on the publisher
+$node_a->safe_psql('postgres', 'CREATE DATABASE db3');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db4');
+
+$node_b->stop;
```

You must wait until all changes have been replicated to the standby here.

12. 040_pg_createsubscriber.pl

Can we do the similar tests without adding new instances? E.g., runs with
dry-run mode and confirms all databases become target.

Fixed the given comments. The attached patch at [1]/messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com contains the
suggested changes.

[1]: /messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#18Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#15)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Feb 6, 2025 at 9:39 AM Peter Smith <smithpb2250@gmail.com> wrote:

Hi Shubham.

Here are some review comments for v3-0001.

FYI, I skipped the review of the test code because I saw Kuroda-san
had already posted some comments about the tests

======
Commit message

1.
This patch enhances the 'pg_createsubscriber' utility to automatically
fetch all non-template databases from the publisher and create subscriptions
for them. This simplifies converting a physical standby to a logical subscriber
for multiple databases, particularly during upgrades.

~

AFAIK you are not creating the subscription at the publisher database,
which is what this description sounds like it is saying. I think you
are creating the subscriptions on the target server on the *same name*
databases that you found on the source server (???). Anyway, this
needs clarification. Also, in some of the comments below, I had the
same confusion.

~~~

2.
A new function 'fetch_all_databases' in 'pg_createsubscriber.c'
that queries the publisher for all databases and adds them to the subscription
options is added.
Additionally, 'validate_databases' ensures that conflicting options are not
used together, enforcing correct usage of '--all-databases'.

~

IMO it is better here to describe the overall logic/algorithms rather
than saying "This new function does this, and this other new function
does that". (otherwise, your commit message will quickly become stale
and need changing all the time).

~~~

3.
The '--all-databases' option fetches all databases from the publisher.
It auto-generates publication and replication slot names as 'pub_names' and
'slot_names' respectively.

~

'pub_names'?
'slot_names'?

What are those strings? Those are not the names that you wrote in the document.

~~~

4.
This option validates database name lengths to ensure generated names fit
within system limits.

~

No. Maybe the patch code ensures this (but I didn't find this
validation anywhere), but certainly "This option" doesn't.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

5.
The synopsis has no mention of --all-databases.

I'm not sure of the best way to do it -- if it becomes too messy to
combine everything then maybe having several is the way to go:

e.g.

pg_createsubscriber [option...] { -a | --all-databases } { -D |
--pgdata }datadir { -P | --publisher-server }connstr
pg_createsubscriber [option...] { -d | --database }dbname { -D |
--pgdata }datadir { -P | --publisher-server }connstr

~~~

6.
+      <para>
+       Automatically fetch all non-template databases from the publisher and
+       create subscriptions for them.
+       When using <option>--all-databases</option>, publication and
+       replication slot names will be automatically generated in the format
+       <literal><replaceable>database</replaceable>_pub</literal> and
+       <literal><replaceable>database</replaceable>_slot</literal>
respectively.
+       When using <option>-a</option>, ensure database names are shorter
+       than 59 characters to allow for generated publication and slot names.
+       replica.
+      </para>

6a.
"fetch all non-template databases from the publisher and create
subscriptions for them" -- Hmm. It's too confusing. I guess you meant
that you are enumerating all the databases on the source server but
then creating subscriptions on all the databases at the target server
that have the same name (???) Anyway, the current text is not clear. I
also think it is better to use the "target server" and "source server"
(aka publication-server) terminology.

~

6b.
I don't think you need to say "When using
<option>--all-databases</option>," because this entire description for
'--all-databases'

~

6c.
Talks about publication and slot names but no mention of subscription names?

~

6d.
That limitation about the 59 char limit seems a strange thing to have
documented here. I wonder why you need to say anything at all. It
might be better if the code just validates all this up-front before
processing and can raise an ERROR in the very rare times this might
happen.

~

6e.
Spurious text junk? "replica"

~

6f.
Don't you need to also say something like "Cannot be used with
'--publication', '--subscription', or '--replication-slot'."

~~~

7. --database

-       switches.
+       switches. If <option>-d</option> and <option>-a</option> are specified
+       together, <application>pg_createsubscriber</application> will return
+       an error.

A nicer way to say that would be to spell it out fully. e.g. "Cannot
be used with --all-databases'.".

~~~

8.
For --publication don't you need to mention "Cannot be used with
--all-databases'.".

~~~

9.
For --subscription don't you need to mention "Cannot be used with
--all-databases'.".

~~~

10.
For --replication-slot don't you need to mention "Cannot be used with
--all-databases'.".

======
src/bin/pg_basebackup/pg_createsubscriber.c

usage:

11.
printf(_("\nOptions:\n"));
+ printf(_(" -a, --all-databases fetch and specify all
databases\n"));

This doesn't really seem a helpful usage text. Should it say something
more like "create subscriptions for all databases"... (??) **

** I have difficulty imagining what is the expected behaviour in cases
where there might be different databases at the source server and
target server. I think your documentation needs to have some special
notes about this to make it clear what server you are enumerating
these databases on, and then what happens if there is some mismatch of
what databases are available on the *other* server.

~~~

fetch_all_databases:

12.
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_all_databases(struct CreateSubscriberOptions *opt)
+{

12a.
What is a "placeholder function"?

~

12b.
I would be nicers to use the same terminology that seems more common
in the documentation -- e.g. "source server" instead of "publisher".
Also, the function name should make it clear if you are getting these
data from the source server or target server.

~~~

13.
+ /* Process the query result */
+ num_rows = PQntuples(res);
+ for (int i = 0; i < num_rows; i++)
+ {
+ const char *dbname = PQgetvalue(res, i, 0);
+
+ simple_string_list_append(&opt->database_names, dbname);
+ num_dbs++;
+ }
The increment num_dbs++ is a bit sneaky. AFAICT you are effectively
pretending --all-databases is the equivalent of a whole lot of
--database so the subsequent logic won't know the difference. I think
that logic deserves a code comment.

~~~

14.
+ /* Check if any databases were added */
+ if (opt->database_names.head == NULL)
+ {
+ pg_log_error("no database names could be fetched or specified");
+ pg_log_error_hint("Ensure that databases exist on the publisher or
specify a database using --database option.");
+ PQclear(res);
+ PQfinish(conn);
+ exit(1);
+ }
+
+ PQclear(res);
+ PQfinish(conn);
+}

14a.
The "were added" seems odd. I think you mean. Just "Error if no
databases were found".

~

14b.
Isn't it simpler just to check if num_dbs == 0?

~

14c.
"no database names could be fetched or specified" seems like a weird
error text. e.g. "specified"?

~

14d.
The hint seems strange to me. If there are no databases found using
--all-database then how will "specify a database using --database
option." help the user get a better result?

~~~

validate_databases:

15.
+static void
+validate_databases(struct CreateSubscriberOptions *opt)
+{
+ /* Check for conflicting options */
+ if (opt->all_databases && opt->database_names.head != NULL)
+ {
+ pg_log_error("cannot specify --dbname when using --all-databases");
+ exit(1);
+ }
+
+ /*
+ * Ensure publication and slot names are not manually specified with
+ * --all-databases
+ */
+ if (opt->all_databases &&
+ (opt->pub_names.head != NULL || opt->replslot_names.head != NULL))
+ {
+ pg_log_error("cannot specify --publication or --replication-slot
when using --all-databases");
+ exit(1);
+ }
+

I think all this switch-checking is misplaced. It should be happening
up-front in the main function.

~~~

16.
+ /* Auto-generate publication and slot names for all databases */
+ if (opt->all_databases)
+ {
+ SimpleStringListCell *cell;
+
+ fetch_all_databases(opt);
+
+ cell = opt->database_names.head;
+
+ while (cell != NULL)
+ {
+ char slot_name[NAMEDATALEN];
+
+ snprintf(slot_name, sizeof(slot_name), "%s_slot", cell->val);
+ simple_string_list_append(&opt->replslot_names, slot_name);
+
+ snprintf(slot_name, sizeof(slot_name), "%s_pub", cell->val);
+ simple_string_list_append(&opt->pub_names, slot_name);
+
+ cell = cell->next;
+ }
+ }

16a.
Why is this code checking 'opt->all_databases'? Isn't it only possible
to get to this function via --all-databases. You can just Assert this.

~

16b.
Why are you reusing 'slot_name' variable like this?

~

16c.
Was there some ERROR checking missing here to ensure the length of the
dbname is not going to cause pub/slot to exceed NAMEDATALEN?

~

16d.
In hindsight, most of this function seems unnecessary to me. Probably
you could've done all this pub/slot name assignment inside the
fetch_all_databases() at the same time as you were doing
simple_string_list_append(&opt->database_names, dbname); for each
database.

~~~

17.
- while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+ while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
long_options, &option_index)) != -1)

Isn't the code within this loop missing all the early switch
validation for checking you are not combining incompatible switches?

e.g.
--all-databases - "Cannot specify --all-databases more than once"
--all-databases - "Cannot combine with --database, --publication,
--subscription, --replication-slot"
--database - "Cannot combine with --all-databases"
--publication - "Cannot combine with --all-databases"
--subscription - "Cannot combine with --all-databases"
--replication-slot - "Cannot combine with --all-databases"

~~~

18.
+
+
/* Number of object names must match number of databases */

Remove spurious whitespace.

======

Fixed the given comments. The attached patch at [1]/messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com contains the
suggested changes.

[1]: /messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#19Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#17)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Fixed the given comments. The attached patch at [1] contains the
suggested changes.

Thanks for updates. I registered the thread to the commitfest entry [1]https://commitfest.postgresql.org/52/5566/.
Few comments.

01. fetch_source_databases

```
+ const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
```

You told given comments were fixed, but you ignored this. Please address it
or clarify the reason why you didn't. My comment was:

We should verify the attribute datallowconn, which indicates we can connect to the
database. If it is false, no one would be able to connect to the db and define anything.

02. fetch_source_databases
```
+       /* Check for connection errors */
+       if (PQstatus(conn) != CONNECTION_OK)
+       {
+               pg_log_error("connection to the source server failed: %s", PQerrorMessage(conn));
+               disconnect_database(conn, false);
+               exit(1);
+       }
```

connect_database() does the error handling, no need to do manually.

03. fetch_source_databases
```
+ pg_log_error("failed to execute query on the source server: %s", PQerrorMessage(conn));
```

I feel the error message is too general. Also, PQerrorMessage() is not suitable for getting error messages
generated by the query. How about:
pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage());

04. fetch_source_databases
```
+       /* Error if no databases were found on the source server */
+       if (num_rows == 0)
+       {
+               pg_log_error("no databases found on the source server");
+               pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+               PQclear(res);
+               disconnect_database(conn, false);
+               exit(1);
+       }
```

Can you clarify when this happens?

05. main

```
                        case 'd':
+                               if (opt.all_databases)
+                               {
+                                       pg_log_error("--database cannot be used with --all-databases");
+                                       exit(1);
+
```

I think this erroring is not enough. getopt_long() receives options with the specified ordering, thus
-d can be accepted if the -a is specified latter.
(Same thing can be said for 'P', '--replslot' and '--subscription'.)

pg_createsubscriber ... -a -d postgres -> can be rejected,
pg_createsubscriber ... -d postgres -a -> cannot be rejected.

To avoid the issue, you must 1) add if-statements in 'a' case or 2) do validation outside of the
switch-case. I prefer 2.

[1]: https://commitfest.postgresql.org/52/5566/

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#20Peter Smith
smithpb2250@gmail.com
In reply to: Shubham Khanna (#16)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Some review comments for v4-0001.

======
Commit message

1.
This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option, which automatically fetches all non-template
databases on the source server (publisher) and creates corresponding
subscriptions on the target server (subscriber). This simplifies the process
of converting a physical standby to a logical subscriber, particularly
during upgrades.

When '--all-databases' is used, the tool queries the source server for all
databases and attempts to create subscriptions on the target server for
databases with matching names. The tool auto-generates subscription,publication
and replication slot names using the format:
- Subscription: '<database>_sub'
- Publication: '<database>_pub'
- Replication slot: '<database>_slot'

~

There seems a lot of duplication in the above 2 paragraphs.
Duplication can be removed.
BTW, those generated formats are suspicious -- see a later comment below.

SUGGESTION:
This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The tool auto-generates subscription, publication and replication slot
names using the format:
...

~~~

2.
The patch ensures that conflicting options such as '--all-databases' and
'--database', '--publication', '--subscription', or '--replication-slot' cannot
be used together

~

That wording seems misleading because it makes it sound like
'--publication' and '--database' cannot be used together.

SUGGESTION
The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is
specified.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

3.
What about the synopsis (??)

v4 did not address my previous review comment [1, #5] about the synopsis

5.
The synopsis has no mention of --all-databases.

I'm not sure of the best way to do it -- if it becomes too messy to
combine everything then maybe having several is the way to go:

e.g.

pg_createsubscriber [option...] { -a | --all-databases } { -D |
--pgdata }datadir { -P | --publisher-server }connstr
pg_createsubscriber [option...] { -d | --database }dbname { -D |
--pgdata }datadir { -P | --publisher-server }connstr

~~~

4.
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, it is
+       skipped or an error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.

"it is skipped or an error is raised" (??) So which is it? It cannot be both.

~~~

5.
+       Subscription names, publication names, and replication slot names are
+       automatically generated using the format:
+       <literal><replaceable>database</replaceable>_sub</literal>
+       <literal><replaceable>database</replaceable>_pub</literal> and
+       <literal><replaceable>database</replaceable>_slot</literal>
respectively.
+      </para>

This seems strange to me, because according to the documentation NOTES
section when --database is used and there is no
publication/subscription name etc then all the generated name format
looks like:

------
If the --publication option is not specified, the publication has the
following name pattern: “pg_createsubscriber_%u_%x” (parameter:
database oid, random int). If the --replication-slot option is not
specified, the replication slot has the following name pattern:
“pg_createsubscriber_%u_%x” (parameters: database oid, random int).
------

and

------
If the --subscription option is not specified, the subscription has
the following name pattern: “pg_createsubscriber_%u_%x” (parameters:
database oid, random int).
------

So, why are the generated names different for '--all-databases'?
Actually, I can't see any code even doing what the new documentation
is saying so is the documentation even telling the truth? The commit
message will also be impacted.

~~~

6.
+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--publication</option>, <option>--subscription</option>,
+       or <option>--replication-slot</option>.
+      </para>

What about "--database".

~~~

7.
For all the other incompatible options you wrote:

"Cannot be used together with <option>-a</option>."

IMO it might be nioer to give the full name of the option.
e.g. "Cannot be used together with <option>--all-databases</option>."

======
src/bin/pg_basebackup/pg_createsubscriber.c

usage:

8.
+ printf(_(" -a, --all-databases create subscriptions for
all matching databases on the target\n"));

That seems ambiguous to me. How about something more like below to
clarify the subscription is created on the target.

"for all source databases create subscriptions on the same target database"

~~~

fetch_source_databases:

9.
+ const char *query = "SELECT datname FROM pg_database WHERE
datistemplate = false";

Do we need this variable? It seems simpler/better just to put the SQL in-line.

~~~

main:

10.
  switch (c)
  {
+ case 'a':
+ opt.all_databases = true;
+ break;
+

10a.
Missing the check for duplicate option --all-databases. OTOH maybe you
don't care about this error, because probably it is harmless if you
just ignore it.

~

10b.
Missing the check for --database, --publication, --subscription,
--replication-slot.

(e.g. if user specified the --all-databases *after* those other options)

~~~

11.
+ if (opt.all_databases)
+ {
+ pg_log_error("--replslot cannot be used with --all-databases");
+ exit(1);
+ }

Where'd this name come from? AFAICT there is no such option as "--replslot"

~~~

12.
- * If --database option is not provided, try to obtain the dbname from
- * the publisher conninfo. If dbname parameter is not available, error
- * out.
+ * If neither --database nor --all-databases option is provided, try
+ * to obtain the dbname from the publisher conninfo. If dbname
+ * parameter is not available, error out.

For this comment to make sense I think the previous comment (where
fetch_source_databases is called) needs to explain that when
--all-databases is defined then it is going to treat that internally
as the equivalent of a whole lot of user-specified --database options.

======
.../t/040_pg_createsubscriber.pl

13.
+ qr/cannot specify --publication or --replication-slot when using
--all-databases/,
+ 'fail if --all-databases is used with publication and slot');
+

How are tests expecting this even passing?

Searching the patch I cannot find any such error!

======
[1]: /messages/by-id/CAHut+PuyBsOJTSygus2-yp60sw_phwYQ-JyC+U6fCBMos9x2LA@mail.gmail.com

Kind Regards,
Peter Smith.
Fujitsu Australia

#21Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#19)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Feb 10, 2025 at 6:58 AM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Fixed the given comments. The attached patch at [1] contains the
suggested changes.

Thanks for updates. I registered the thread to the commitfest entry [1].
Few comments.

01. fetch_source_databases

```
+ const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
```

You told given comments were fixed, but you ignored this. Please address it
or clarify the reason why you didn't. My comment was:

We should verify the attribute datallowconn, which indicates we can connect to the
database. If it is false, no one would be able to connect to the db and define anything.

Fixed.

02. fetch_source_databases
```
+       /* Check for connection errors */
+       if (PQstatus(conn) != CONNECTION_OK)
+       {
+               pg_log_error("connection to the source server failed: %s", PQerrorMessage(conn));
+               disconnect_database(conn, false);
+               exit(1);
+       }
```

connect_database() does the error handling, no need to do manually.

Fixed.

03. fetch_source_databases
```
+ pg_log_error("failed to execute query on the source server: %s", PQerrorMessage(conn));
```

I feel the error message is too general. Also, PQerrorMessage() is not suitable for getting error messages
generated by the query. How about:
pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage());

Fixed.

04. fetch_source_databases
```
+       /* Error if no databases were found on the source server */
+       if (num_rows == 0)
+       {
+               pg_log_error("no databases found on the source server");
+               pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+               PQclear(res);
+               disconnect_database(conn, false);
+               exit(1);
+       }
```

Can you clarify when this happens?

This can happen when the postgres database has been dropped by
connecting to a template database.

05. main

```
case 'd':
+                               if (opt.all_databases)
+                               {
+                                       pg_log_error("--database cannot be used with --all-databases");
+                                       exit(1);
+
```

I think this erroring is not enough. getopt_long() receives options with the specified ordering, thus
-d can be accepted if the -a is specified latter.
(Same thing can be said for 'P', '--replslot' and '--subscription'.)

pg_createsubscriber ... -a -d postgres -> can be rejected,
pg_createsubscriber ... -d postgres -a -> cannot be rejected.

Fixed.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v5-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v5-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 313f88c2801522da174cddce343bc8947fc393ff Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v5] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  38 +++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 124 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  98 ++++++++++++++
 3 files changed, 252 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..e669c8250c 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -24,6 +24,10 @@ PostgreSQL documentation
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
    <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all-databases</option></arg>
+    </group>
     <group choice="req">
      <arg choice="plain"><option>-d</option></arg>
      <arg choice="plain"><option>--database</option></arg>
@@ -87,6 +91,29 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, an
+       error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
+       Subscription names, publication names, and replication slot names are
+       automatically generated.
+      </para>
+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +121,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +241,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>-a</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +258,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +274,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>-a</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..39a28134c0 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* fetch and specify all databases */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             create subscriptions on the target for all source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1860,11 +1863,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all-databases is specified, fetch a list of all user-created databases
+ * from the source server. Internally, this is treated as if the user specified
+ * multiple --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_rows == 0)
+	{
+		pg_log_error("no databases found on the source server");
+		pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1927,6 +1983,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1949,12 +2006,49 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				if (opt.all_databases)
+				{
+					pg_log_error("--all-databases specified multiple times");
+					exit(1);
+				}
+
+				if (num_dbs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --database");
+					exit(1);
+				}
+
+				if (num_pubs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --publication");
+					exit(1);
+				}
+
+				if (num_replslots > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --replication-slot");
+					exit(1);
+				}
+
+				if (num_subs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --subscription");
+					exit(1);
+				}
+				opt.all_databases = true;
+				break;
 			case 'd':
+				if (opt.all_databases)
+				{
+					pg_log_error("--database cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
 					simple_string_list_append(&opt.database_names, optarg);
@@ -1977,6 +2071,11 @@ main(int argc, char **argv)
 				opt.sub_port = pg_strdup(optarg);
 				break;
 			case 'P':
+				if (opt.all_databases)
+				{
+					pg_log_error("--publication cannot be used with --all-databases");
+					exit(1);
+				}
 				opt.pub_conninfo_str = pg_strdup(optarg);
 				break;
 			case 's':
@@ -2008,6 +2107,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 3:
+				if (opt.all_databases)
+				{
+					pg_log_error("--replication-slot cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.replslot_names, optarg))
 				{
 					simple_string_list_append(&opt.replslot_names, optarg);
@@ -2020,6 +2124,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 4:
+				if (opt.all_databases)
+				{
+					pg_log_error("--subscription cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.sub_names, optarg))
 				{
 					simple_string_list_append(&opt.sub_names, optarg);
@@ -2091,14 +2200,21 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all-databases is
+	 * specified.
+	 */
+	if (opt.all_databases)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all-databases option is provided, try
+		 * to obtain the dbname from the publisher conninfo. If dbname
+		 * parameter is not available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..6bc255e137 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,104 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with specifying '--all-databases' twice and verify
+# the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases', '--all-databases',
+	],
+	qr/--all-databases specified multiple times/,
+	'fail if --all-databases is used more than once');
+
+# run pg_createsubscriber with '--dbname' and '--all-databases' and verify the
+# failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--dbname' => 'db1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --dbname/,
+	'fail if --all-databases is used with --dbname');
+
+# run pg_createsubscriber with '--publication' and '--all-databases' and verify
+# the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --publication/,
+	'fail if --all-databases is used with --publication');
+
+# run pg_createsubscriber with '--replication-slot' and '--all-databases' and
+# verify the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --replication-slot/,
+	'fail if --all-databases is used with --replication-slot');
+
+# run pg_createsubscriber with '--subscription' and '--all-databases' and
+# verify the failure
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--subscription' => 'sub1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --subscription/,
+	'fail if --all-databases is used with --subscription');
+
+# run pg_createsubscriber with '--all-databases' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
-- 
2.41.0.windows.3

#22Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#20)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Feb 10, 2025 at 7:05 AM Peter Smith <smithpb2250@gmail.com> wrote:

Some review comments for v4-0001.

======
Commit message

1.
This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option, which automatically fetches all non-template
databases on the source server (publisher) and creates corresponding
subscriptions on the target server (subscriber). This simplifies the process
of converting a physical standby to a logical subscriber, particularly
during upgrades.

When '--all-databases' is used, the tool queries the source server for all
databases and attempts to create subscriptions on the target server for
databases with matching names. The tool auto-generates subscription,publication
and replication slot names using the format:
- Subscription: '<database>_sub'
- Publication: '<database>_pub'
- Replication slot: '<database>_slot'

~

There seems a lot of duplication in the above 2 paragraphs.
Duplication can be removed.
BTW, those generated formats are suspicious -- see a later comment below.

SUGGESTION:
This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The tool auto-generates subscription, publication and replication slot
names using the format:
...

~~~

2.
The patch ensures that conflicting options such as '--all-databases' and
'--database', '--publication', '--subscription', or '--replication-slot' cannot
be used together

~

That wording seems misleading because it makes it sound like
'--publication' and '--database' cannot be used together.

SUGGESTION
The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is
specified.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

3.
What about the synopsis (??)

v4 did not address my previous review comment [1, #5] about the synopsis

5.
The synopsis has no mention of --all-databases.

I'm not sure of the best way to do it -- if it becomes too messy to
combine everything then maybe having several is the way to go:

e.g.

pg_createsubscriber [option...] { -a | --all-databases } { -D |
--pgdata }datadir { -P | --publisher-server }connstr
pg_createsubscriber [option...] { -d | --database }dbname { -D |
--pgdata }datadir { -P | --publisher-server }connstr

~~~

4.
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, it is
+       skipped or an error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.

"it is skipped or an error is raised" (??) So which is it? It cannot be both.

~~~

5.
+       Subscription names, publication names, and replication slot names are
+       automatically generated using the format:
+       <literal><replaceable>database</replaceable>_sub</literal>
+       <literal><replaceable>database</replaceable>_pub</literal> and
+       <literal><replaceable>database</replaceable>_slot</literal>
respectively.
+      </para>

This seems strange to me, because according to the documentation NOTES
section when --database is used and there is no
publication/subscription name etc then all the generated name format
looks like:

------
If the --publication option is not specified, the publication has the
following name pattern: “pg_createsubscriber_%u_%x” (parameter:
database oid, random int). If the --replication-slot option is not
specified, the replication slot has the following name pattern:
“pg_createsubscriber_%u_%x” (parameters: database oid, random int).
------

and

------
If the --subscription option is not specified, the subscription has
the following name pattern: “pg_createsubscriber_%u_%x” (parameters:
database oid, random int).
------

So, why are the generated names different for '--all-databases'?
Actually, I can't see any code even doing what the new documentation
is saying so is the documentation even telling the truth? The commit
message will also be impacted.

~~~

6.
+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--publication</option>, <option>--subscription</option>,
+       or <option>--replication-slot</option>.
+      </para>

What about "--database".

~~~

7.
For all the other incompatible options you wrote:

"Cannot be used together with <option>-a</option>."

IMO it might be nioer to give the full name of the option.
e.g. "Cannot be used together with <option>--all-databases</option>."

======
src/bin/pg_basebackup/pg_createsubscriber.c

usage:

8.
+ printf(_(" -a, --all-databases create subscriptions for
all matching databases on the target\n"));

That seems ambiguous to me. How about something more like below to
clarify the subscription is created on the target.

"for all source databases create subscriptions on the same target database"

~~~

fetch_source_databases:

9.
+ const char *query = "SELECT datname FROM pg_database WHERE
datistemplate = false";

Do we need this variable? It seems simpler/better just to put the SQL in-line.

~~~

main:

10.
switch (c)
{
+ case 'a':
+ opt.all_databases = true;
+ break;
+

10a.
Missing the check for duplicate option --all-databases. OTOH maybe you
don't care about this error, because probably it is harmless if you
just ignore it.

~

10b.
Missing the check for --database, --publication, --subscription,
--replication-slot.

(e.g. if user specified the --all-databases *after* those other options)

~~~

11.
+ if (opt.all_databases)
+ {
+ pg_log_error("--replslot cannot be used with --all-databases");
+ exit(1);
+ }

Where'd this name come from? AFAICT there is no such option as "--replslot"

~~~

12.
- * If --database option is not provided, try to obtain the dbname from
- * the publisher conninfo. If dbname parameter is not available, error
- * out.
+ * If neither --database nor --all-databases option is provided, try
+ * to obtain the dbname from the publisher conninfo. If dbname
+ * parameter is not available, error out.

For this comment to make sense I think the previous comment (where
fetch_source_databases is called) needs to explain that when
--all-databases is defined then it is going to treat that internally
as the equivalent of a whole lot of user-specified --database options.

======
.../t/040_pg_createsubscriber.pl

13.
+ qr/cannot specify --publication or --replication-slot when using
--all-databases/,
+ 'fail if --all-databases is used with publication and slot');
+

How are tests expecting this even passing?

Searching the patch I cannot find any such error!

======

Fixed the given comments. The attached patch at [1]/messages/by-id/CAHv8Rj+mhYgh8XLFM8sN8J05z0ia+knfB1kP6kjbnB55H-b-mw@mail.gmail.com contains the
suggested changes.

[1]: /messages/by-id/CAHv8Rj+mhYgh8XLFM8sN8J05z0ia+knfB1kP6kjbnB55H-b-mw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#23vignesh C
vignesh21@gmail.com
In reply to: Shubham Khanna (#21)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, 10 Feb 2025 at 20:36, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patch contains the suggested changes.

If a new database is created on the primary server while
pg_createsubscriber is running, the subscription will not be created
for the new database.
To reproduce this issue, follow these steps:
Debug pg_createsubscriber and set a breakpoint after the
fetch_source_databases function, where the databases would have been
prepared.
While execution of pg_createsubscriber is paused, create a new
database on the primary node.
You will observe that the database is created on the standby node, but
the subscription will not be created for the newly created database.
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+       PGconn     *conn;
+       PGresult   *res;
+       int                     num_rows;
+
+       /* Establish a connection to the PostgreSQL server */
+       conn = connect_database(opt->pub_conninfo_str, true);
+
+       res = PQexec(conn, "SELECT datname FROM pg_database WHERE
datistemplate = false AND datallowconn = true");
+
+       /* Check for errors during query execution */
+       if (PQresultStatus(res) != PGRES_TUPLES_OK)

Regards,
Vignesh

#24Peter Smith
smithpb2250@gmail.com
In reply to: Shubham Khanna (#21)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi Shubham.

Responding with a blanket statement "Fixed the given comments" makes
it difficult to be sure what was done when I come to confirm the
changes. Often embedded questions go unanswered, and if changes are
*not* made, then I can't tell if there was a good reason for rejection
or was the comment just overlooked. Please can you treat the itemised
feedback as a checklist and reply individually to each item. Even just
saying "Fixed" is useful because then I can trust the item was seen
and addressed.

e.g. From previous review [1]/messages/by-id/CAHut+PuYmTyi6kPFnJDTvD=aLHd0kyX4VG6iDQVEk-ixov1AwA@mail.gmail.com
#7. Not fixed. The same docs issue still exists for --publication and
for --subscription.
#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

//////

Some more review comments for the patch v5-0001.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

Synopsis

1.
pg_createsubscriber [option...] { -a | --all-databases } { -d |
--database }dbname { -D | --pgdata }datadir { -P | --publisher-server
}connstr

This looks misleading because '-all-database' and '--database' cannot
be combined.

I think it will be better to have 2 completely different synopsis like
I had originally suggested [2, comment #5]. E.g. just add a whole
seperate <cmdsynopsis> block adjacent to the other one in the SGML:

------
<cmdsynopsis>
<command>pg_createsubscriber</command>
<arg rep="repeat"><replaceable>option</replaceable></arg>
<group choice="plain">
<group choice="req">
<arg choice="plain"><option>-a</option></arg>
<arg choice="plain"><option>--all-databases</option></arg>
</group>
<group choice="req">
<arg choice="plain"><option>-D</option> </arg>
<arg choice="plain"><option>--pgdata</option></arg>
</group>
<replaceable>datadir</replaceable>
<group choice="req">
<arg choice="plain"><option>-P</option></arg>
<arg choice="plain"><option>--publisher-server</option></arg>
</group>
<replaceable>connstr</replaceable>
</group>
</cmdsynopsis>
------

~~~

2. --all-databases

+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>

If you want consistent wording with the other places in this patch,
then this should just be the last sentence of the previous paragraph
and be worded like: "Cannot be used together with..."

~~~

3. --publication

-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>-a</option>.

Previously reported. Claimed fixed -a to --all-databases. Not fixed.

~~~

4. --subscription

-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>-a</option>.

(same as the previous review comment).

Previously reported. Claimed fixed -a to --all-databases. Not fixed.

======
src/bin/pg_basebackup/pg_createsubscriber.c

5.
+ bool all_databases; /* fetch and specify all databases */

Comment wording. "and specified" (??). Maybe just "--all-databases
option was specified"

======
[1]: /messages/by-id/CAHut+PuYmTyi6kPFnJDTvD=aLHd0kyX4VG6iDQVEk-ixov1AwA@mail.gmail.com
[2]: /messages/by-id/CAHut+PuyBsOJTSygus2-yp60sw_phwYQ-JyC+U6fCBMos9x2LA@mail.gmail.com

Kind Regards,
Peter Smith.
Fujitsu Australia

#25Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#24)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Feb 11, 2025 at 6:30 AM Peter Smith <smithpb2250@gmail.com> wrote:

Hi Shubham.

Responding with a blanket statement "Fixed the given comments" makes
it difficult to be sure what was done when I come to confirm the
changes. Often embedded questions go unanswered, and if changes are
*not* made, then I can't tell if there was a good reason for rejection
or was the comment just overlooked. Please can you treat the itemised
feedback as a checklist and reply individually to each item. Even just
saying "Fixed" is useful because then I can trust the item was seen
and addressed.

I appreciate the feedback. I will ensure that each item is addressed
individually.

e.g. From previous review [1]
#7. Not fixed. The same docs issue still exists for --publication and
for --subscription.

Fixed this issue in the attached patch.

#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

In the v4-0001 patch [1]/messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com, the tests were mistakenly using
'command_fails' instead of 'command_fails_like' to verify failed test
cases. Since 'command_fails' only checks for a non-zero exit code and
not specific error messages, the tests were passing even when the
expected errors were not being triggered correctly.
To address this, I have updated the patch to use 'command_fails_like',
ensuring that the test cases now explicitly verify the correct failure
messages.

//////

Some more review comments for the patch v5-0001.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

Synopsis

1.
pg_createsubscriber [option...] { -a | --all-databases } { -d |
--database }dbname { -D | --pgdata }datadir { -P | --publisher-server
}connstr

This looks misleading because '-all-database' and '--database' cannot
be combined.

I think it will be better to have 2 completely different synopsis like
I had originally suggested [2, comment #5]. E.g. just add a whole
seperate <cmdsynopsis> block adjacent to the other one in the SGML:

------
<cmdsynopsis>
<command>pg_createsubscriber</command>
<arg rep="repeat"><replaceable>option</replaceable></arg>
<group choice="plain">
<group choice="req">
<arg choice="plain"><option>-a</option></arg>
<arg choice="plain"><option>--all-databases</option></arg>
</group>
<group choice="req">
<arg choice="plain"><option>-D</option> </arg>
<arg choice="plain"><option>--pgdata</option></arg>
</group>
<replaceable>datadir</replaceable>
<group choice="req">
<arg choice="plain"><option>-P</option></arg>
<arg choice="plain"><option>--publisher-server</option></arg>
</group>
<replaceable>connstr</replaceable>
</group>
</cmdsynopsis>
------

~~~

Fixed.

2. --all-databases

+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>

If you want consistent wording with the other places in this patch,
then this should just be the last sentence of the previous paragraph
and be worded like: "Cannot be used together with..."

~~~

Fixed.

3. --publication

-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>-a</option>.

Previously reported. Claimed fixed -a to --all-databases. Not fixed.

~~~

Fixed.

4. --subscription

-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>-a</option>.

(same as the previous review comment).

Previously reported. Claimed fixed -a to --all-databases. Not fixed.

Fixed.

======
src/bin/pg_basebackup/pg_createsubscriber.c

5.
+ bool all_databases; /* fetch and specify all databases */

Comment wording. "and specified" (??). Maybe just "--all-databases
option was specified"

======

Fixed.

The attached Patch contains the suggested changes.

[1]: /messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com

Thanks and regards,
Shubham Khanna.

Attachments:

v6-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v6-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From f39bc6244996a1632efeb5895d2656e53af9d8ca Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v6] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 +++++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 124 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  98 ++++++++++++++
 3 files changed, 266 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..08f33479d4 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all-databases</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,26 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, an
+       error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +135,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +255,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +272,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +288,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..f7687e24f3 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* --all-databases option was specified */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             create subscriptions on the target for all source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1860,11 +1863,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all-databases is specified, fetch a list of all user-created databases
+ * from the source server. Internally, this is treated as if the user specified
+ * multiple --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_rows == 0)
+	{
+		pg_log_error("no databases found on the source server");
+		pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1927,6 +1983,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1949,12 +2006,49 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				if (opt.all_databases)
+				{
+					pg_log_error("--all-databases specified multiple times");
+					exit(1);
+				}
+
+				if (num_dbs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --database");
+					exit(1);
+				}
+
+				if (num_pubs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --publication");
+					exit(1);
+				}
+
+				if (num_replslots > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --replication-slot");
+					exit(1);
+				}
+
+				if (num_subs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --subscription");
+					exit(1);
+				}
+				opt.all_databases = true;
+				break;
 			case 'd':
+				if (opt.all_databases)
+				{
+					pg_log_error("--database cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
 					simple_string_list_append(&opt.database_names, optarg);
@@ -1977,6 +2071,11 @@ main(int argc, char **argv)
 				opt.sub_port = pg_strdup(optarg);
 				break;
 			case 'P':
+				if (opt.all_databases)
+				{
+					pg_log_error("--publication cannot be used with --all-databases");
+					exit(1);
+				}
 				opt.pub_conninfo_str = pg_strdup(optarg);
 				break;
 			case 's':
@@ -2008,6 +2107,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 3:
+				if (opt.all_databases)
+				{
+					pg_log_error("--replication-slot cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.replslot_names, optarg))
 				{
 					simple_string_list_append(&opt.replslot_names, optarg);
@@ -2020,6 +2124,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 4:
+				if (opt.all_databases)
+				{
+					pg_log_error("--subscription cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.sub_names, optarg))
 				{
 					simple_string_list_append(&opt.sub_names, optarg);
@@ -2091,14 +2200,21 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all-databases is
+	 * specified.
+	 */
+	if (opt.all_databases)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all-databases option is provided, try
+		 * to obtain the dbname from the publisher conninfo. If dbname
+		 * parameter is not available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..b35bbc0bce 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,104 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with specifying '--all-databases' twice and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases', '--all-databases',
+	],
+	qr/--all-databases specified multiple times/,
+	'fail if --all-databases is used more than once');
+
+# run pg_createsubscriber with '--database' and '--all-databases' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --database/,
+	'fail if --all-databases is used with --database');
+
+# run pg_createsubscriber with '--publication' and '--all-databases' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --publication/,
+	'fail if --all-databases is used with --publication');
+
+# run pg_createsubscriber with '--replication-slot' and '--all-databases' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --replication-slot/,
+	'fail if --all-databases is used with --replication-slot');
+
+# run pg_createsubscriber with '--subscription' and '--all-databases' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--subscription' => 'sub1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --subscription/,
+	'fail if --all-databases is used with --subscription');
+
+# run pg_createsubscriber with '--all-databases' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
-- 
2.34.1

#26Peter Smith
smithpb2250@gmail.com
In reply to: Shubham Khanna (#25)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Feb 11, 2025 at 9:16 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

In the v4-0001 patch [1], the tests were mistakenly using
'command_fails' instead of 'command_fails_like' to verify failed test
cases. Since 'command_fails' only checks for a non-zero exit code and
not specific error messages, the tests were passing even when the
expected errors were not being triggered correctly.
To address this, I have updated the patch to use 'command_fails_like',
ensuring that the test cases now explicitly verify the correct failure
messages.

Ah, that makes sense. Thanks for sharing the reason. So in fact, it
was a valid concern because the v5 was still carrying over the same
flaw from v4... Anyway, it is good to know it is fixed now in v6.

=====

Some general comments for the patch v6-0001:

Do you need to test every possible bad option combination? It may be
fine because the error will be immediately raised so I expect the
execution overhead to be ~zero.

BTW, your bad option combination tests are only using --all-databases
*after* the other options. Maybe you should mix it up a bit, sometimes
putting it *before* the others as well, because rearranging will cause
different errors.

Everything else now looks good to me.

======
Kind Regards,
Peter Smith.
Fujitsu Australia

#27Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#25)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for updating the patch! I feel the patch has good shape. Here is a small comment.

```
+       /* Error if no databases were found on the source server */
+       if (num_rows == 0)
+       {
+               pg_log_error("no databases found on the source server");
+               pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+               PQclear(res);
+               disconnect_database(conn, false);
+               exit(1);
+       }
```

I think the error message is not accurate. This error can happen when there are
user-created database but it is created as template. How about below:

```
-               pg_log_error("no databases found on the source server");
-               pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+               pg_log_error("no convertable databases found on the source server");
+               pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
```

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#28Peter Smith
smithpb2250@gmail.com
In reply to: Peter Smith (#26)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 12, 2025 at 10:58 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Tue, Feb 11, 2025 at 9:16 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

In the v4-0001 patch [1], the tests were mistakenly using
'command_fails' instead of 'command_fails_like' to verify failed test
cases. Since 'command_fails' only checks for a non-zero exit code and
not specific error messages, the tests were passing even when the
expected errors were not being triggered correctly.
To address this, I have updated the patch to use 'command_fails_like',
ensuring that the test cases now explicitly verify the correct failure
messages.

Ah, that makes sense. Thanks for sharing the reason. So in fact, it
was a valid concern because the v5 was still carrying over the same
flaw from v4... Anyway, it is good to know it is fixed now in v6.

FYI, I've proposed a patch [1]/messages/by-id/CAHut+Pu-umU=CkcSvrqL9eQt85Rv_qfLtekqU-yrjfh=GirUQg@mail.gmail.com that can prevent this mistake from
happening in future.

======
[1]: /messages/by-id/CAHut+Pu-umU=CkcSvrqL9eQt85Rv_qfLtekqU-yrjfh=GirUQg@mail.gmail.com

Kind Regards,
Peter Smith.
Fujitsu Australia.

#29Shlok Kyal
shlok.kyal.oss@gmail.com
In reply to: Shubham Khanna (#25)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, 11 Feb 2025 at 15:46, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Tue, Feb 11, 2025 at 6:30 AM Peter Smith <smithpb2250@gmail.com> wrote:

Hi Shubham.

Responding with a blanket statement "Fixed the given comments" makes
it difficult to be sure what was done when I come to confirm the
changes. Often embedded questions go unanswered, and if changes are
*not* made, then I can't tell if there was a good reason for rejection
or was the comment just overlooked. Please can you treat the itemised
feedback as a checklist and reply individually to each item. Even just
saying "Fixed" is useful because then I can trust the item was seen
and addressed.

I appreciate the feedback. I will ensure that each item is addressed
individually.

e.g. From previous review [1]
#7. Not fixed. The same docs issue still exists for --publication and
for --subscription.

Fixed this issue in the attached patch.

#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

In the v4-0001 patch [1], the tests were mistakenly using
'command_fails' instead of 'command_fails_like' to verify failed test
cases. Since 'command_fails' only checks for a non-zero exit code and
not specific error messages, the tests were passing even when the
expected errors were not being triggered correctly.
To address this, I have updated the patch to use 'command_fails_like',
ensuring that the test cases now explicitly verify the correct failure
messages.

//////

Some more review comments for the patch v5-0001.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

Synopsis

1.
pg_createsubscriber [option...] { -a | --all-databases } { -d |
--database }dbname { -D | --pgdata }datadir { -P | --publisher-server
}connstr

This looks misleading because '-all-database' and '--database' cannot
be combined.

I think it will be better to have 2 completely different synopsis like
I had originally suggested [2, comment #5]. E.g. just add a whole
seperate <cmdsynopsis> block adjacent to the other one in the SGML:

------
<cmdsynopsis>
<command>pg_createsubscriber</command>
<arg rep="repeat"><replaceable>option</replaceable></arg>
<group choice="plain">
<group choice="req">
<arg choice="plain"><option>-a</option></arg>
<arg choice="plain"><option>--all-databases</option></arg>
</group>
<group choice="req">
<arg choice="plain"><option>-D</option> </arg>
<arg choice="plain"><option>--pgdata</option></arg>
</group>
<replaceable>datadir</replaceable>
<group choice="req">
<arg choice="plain"><option>-P</option></arg>
<arg choice="plain"><option>--publisher-server</option></arg>
</group>
<replaceable>connstr</replaceable>
</group>
</cmdsynopsis>
------

~~~

Fixed.

2. --all-databases

+      <para>
+       <option>--all-databases</option> cannot be used with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>

If you want consistent wording with the other places in this patch,
then this should just be the last sentence of the previous paragraph
and be worded like: "Cannot be used together with..."

~~~

Fixed.

3. --publication

-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>-a</option>.

Previously reported. Claimed fixed -a to --all-databases. Not fixed.

~~~

Fixed.

4. --subscription

-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>-a</option>.

(same as the previous review comment).

Previously reported. Claimed fixed -a to --all-databases. Not fixed.

Fixed.

======
src/bin/pg_basebackup/pg_createsubscriber.c

5.
+ bool all_databases; /* fetch and specify all databases */

Comment wording. "and specified" (??). Maybe just "--all-databases
option was specified"

======

Fixed.

The attached Patch contains the suggested changes.

[1] - /messages/by-id/CAHv8Rj+DOQ9vvkKCsmDeFKyM073yRfs47Tn_qywQOdx15pmOMA@mail.gmail.com

Hi Shubham,

I reviewed v6 patch, here are some of my comments:

1. Option 'P' is for "publisher-server".
            case 'P':
+               if (opt.all_databases)
+               {
+                   pg_log_error("--publication cannot be used with
--all-databases");
+                   exit(1);
+               }

So, if we provide the "--all-databases" option and then the
"--publisher-server" option. Then the command pg_createsubscriber is
failing, which is wrong.

2. In case we provide "--all-database" option and then "--publication"
option, the pg_createsubscriber command is running. I think the above
condition should be added at "case 2:" instead of "case 'P':"

Thanks and Regards,
Shlok Kyal

#30Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#26)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 12, 2025 at 5:29 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Tue, Feb 11, 2025 at 9:16 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

In the v4-0001 patch [1], the tests were mistakenly using
'command_fails' instead of 'command_fails_like' to verify failed test
cases. Since 'command_fails' only checks for a non-zero exit code and
not specific error messages, the tests were passing even when the
expected errors were not being triggered correctly.
To address this, I have updated the patch to use 'command_fails_like',
ensuring that the test cases now explicitly verify the correct failure
messages.

Ah, that makes sense. Thanks for sharing the reason. So in fact, it
was a valid concern because the v5 was still carrying over the same
flaw from v4... Anyway, it is good to know it is fixed now in v6.

=====

Some general comments for the patch v6-0001:

Do you need to test every possible bad option combination? It may be
fine because the error will be immediately raised so I expect the
execution overhead to be ~zero.

BTW, your bad option combination tests are only using --all-databases
*after* the other options. Maybe you should mix it up a bit, sometimes
putting it *before* the others as well, because rearranging will cause
different errors.

Everything else now looks good to me.

Fixed the given comments. The attached Patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v7-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v7-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From b8695a90f0013a19d677e84e91e66fb3b1c83c4d Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v7] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 +++++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 124 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  98 ++++++++++++++
 3 files changed, 266 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..08f33479d4 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all-databases</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,26 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, an
+       error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +135,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +255,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +272,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +288,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..02d9f24589 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* --all-databases option was specified */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             create subscriptions on the target for all source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1860,11 +1863,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all-databases is specified, fetch a list of all user-created databases
+ * from the source server. Internally, this is treated as if the user specified
+ * multiple --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_rows == 0)
+	{
+		pg_log_error("no convertable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1927,6 +1983,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1949,12 +2006,49 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				if (opt.all_databases)
+				{
+					pg_log_error("--all-databases specified multiple times");
+					exit(1);
+				}
+
+				if (num_dbs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --database");
+					exit(1);
+				}
+
+				if (num_pubs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --publication");
+					exit(1);
+				}
+
+				if (num_replslots > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --replication-slot");
+					exit(1);
+				}
+
+				if (num_subs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --subscription");
+					exit(1);
+				}
+				opt.all_databases = true;
+				break;
 			case 'd':
+				if (opt.all_databases)
+				{
+					pg_log_error("--database cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
 					simple_string_list_append(&opt.database_names, optarg);
@@ -1996,6 +2090,11 @@ main(int argc, char **argv)
 				opt.config_file = pg_strdup(optarg);
 				break;
 			case 2:
+				if (opt.all_databases)
+				{
+					pg_log_error("--publication cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.pub_names, optarg))
 				{
 					simple_string_list_append(&opt.pub_names, optarg);
@@ -2008,6 +2107,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 3:
+				if (opt.all_databases)
+				{
+					pg_log_error("--replication-slot cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.replslot_names, optarg))
 				{
 					simple_string_list_append(&opt.replslot_names, optarg);
@@ -2020,6 +2124,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 4:
+				if (opt.all_databases)
+				{
+					pg_log_error("--subscription cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.sub_names, optarg))
 				{
 					simple_string_list_append(&opt.sub_names, optarg);
@@ -2091,14 +2200,21 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all-databases is
+	 * specified.
+	 */
+	if (opt.all_databases)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all-databases option is provided, try
+		 * to obtain the dbname from the publisher conninfo. If dbname
+		 * parameter is not available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..33a6c03bf2 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,104 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with specifying '--all-databases' twice and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases', '--all-databases',
+	],
+	qr/--all-databases specified multiple times/,
+	'fail if --all-databases is used more than once');
+
+# run pg_createsubscriber with '--database' and '--all-databases' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --database/,
+	'fail if --all-databases is used with --database');
+
+# run pg_createsubscriber with '--publication' and '--all-databases' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all-databases/,
+	'fail if --all-databases is used with --publication');
+
+# run pg_createsubscriber with '--replication-slot' and '--all-databases' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --replication-slot/,
+	'fail if --all-databases is used with --replication-slot');
+
+# run pg_createsubscriber with '--subscription' and '--all-databases' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all-databases/,
+	'fail if --all-databases is used with --subscription');
+
+# run pg_createsubscriber with '--all-databases' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
-- 
2.34.1

#31Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#27)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 12, 2025 at 6:46 AM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch! I feel the patch has good shape. Here is a small comment.

```
+       /* Error if no databases were found on the source server */
+       if (num_rows == 0)
+       {
+               pg_log_error("no databases found on the source server");
+               pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+               PQclear(res);
+               disconnect_database(conn, false);
+               exit(1);
+       }
```

I think the error message is not accurate. This error can happen when there are
user-created database but it is created as template. How about below:

```
-               pg_log_error("no databases found on the source server");
-               pg_log_error_hint("Ensure that there are user-created databases on the source server.");
+               pg_log_error("no convertable databases found on the source server");
+               pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
```

Fixed the given comment. The attached patch at [1]/messages/by-id/CAHv8RjKc6LMJ86b4yCmwNTn0c65mz0BGMCF-vPJSKDMOGagVGA@mail.gmail.com contains the
suggested changes.

[1]: /messages/by-id/CAHv8RjKc6LMJ86b4yCmwNTn0c65mz0BGMCF-vPJSKDMOGagVGA@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#32Shubham Khanna
khannashubham1197@gmail.com
In reply to: Shubham Khanna (#30)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 12, 2025 at 1:15 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Wed, Feb 12, 2025 at 5:29 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Tue, Feb 11, 2025 at 9:16 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

#13. Unanswered question "How are tests expecting this even passing?".
Was a reason identified? IOW, how can we be sure the latest tests
don't have a similar problem?

In the v4-0001 patch [1], the tests were mistakenly using
'command_fails' instead of 'command_fails_like' to verify failed test
cases. Since 'command_fails' only checks for a non-zero exit code and
not specific error messages, the tests were passing even when the
expected errors were not being triggered correctly.
To address this, I have updated the patch to use 'command_fails_like',
ensuring that the test cases now explicitly verify the correct failure
messages.

Ah, that makes sense. Thanks for sharing the reason. So in fact, it
was a valid concern because the v5 was still carrying over the same
flaw from v4... Anyway, it is good to know it is fixed now in v6.

=====

Some general comments for the patch v6-0001:

Do you need to test every possible bad option combination? It may be
fine because the error will be immediately raised so I expect the
execution overhead to be ~zero.

BTW, your bad option combination tests are only using --all-databases
*after* the other options. Maybe you should mix it up a bit, sometimes
putting it *before* the others as well, because rearranging will cause
different errors.

Everything else now looks good to me.

Fixed the comment given by Shlok at [1]/messages/by-id/CANhcyEXGwFeQd2fuB2txm1maCC2zKyROUR5exEMGzPYdrZ4CPQ@mail.gmail.com. The attached patch at [2]/messages/by-id/CAHv8RjKc6LMJ86b4yCmwNTn0c65mz0BGMCF-vPJSKDMOGagVGA@mail.gmail.com
contains the suggested changes.

[1]: /messages/by-id/CANhcyEXGwFeQd2fuB2txm1maCC2zKyROUR5exEMGzPYdrZ4CPQ@mail.gmail.com
[2]: /messages/by-id/CAHv8RjKc6LMJ86b4yCmwNTn0c65mz0BGMCF-vPJSKDMOGagVGA@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#33Peter Smith
smithpb2250@gmail.com
In reply to: Shubham Khanna (#30)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Some review comments for v7-0001

======
src/bin/pg_basebackup/pg_createsubscriber.c

1.
+ /* Error if no databases were found on the source server */
+ if (num_rows == 0)
+ {
+ pg_log_error("no convertable databases found on the source server");
+ pg_log_error_hint("Ensure that there are non-template and
connectable databases on the source server.");
+ PQclear(res);
+ disconnect_database(conn, false);
+ exit(1);
+ }

1a.
The spelling is "convertible", not "convertable"

~

1b.
What does "convertible databases" mean here, and will that term make
sense to a user? How are we "converting" the source server databases;
AFAIK we are simply connecting to them and adding publications.

IMO a better choice of adjectives can be found below.
"no suitable databases found..."
"no appropriate databases found..."
"no eligible databases found..."
etc.

======
Kind Regards,
Peter Smith.
Fujitsu Australia.

#34Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#33)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Feb 13, 2025 at 4:48 AM Peter Smith <smithpb2250@gmail.com> wrote:

Some review comments for v7-0001

======
src/bin/pg_basebackup/pg_createsubscriber.c

1.
+ /* Error if no databases were found on the source server */
+ if (num_rows == 0)
+ {
+ pg_log_error("no convertable databases found on the source server");
+ pg_log_error_hint("Ensure that there are non-template and
connectable databases on the source server.");
+ PQclear(res);
+ disconnect_database(conn, false);
+ exit(1);
+ }

1a.
The spelling is "convertible", not "convertable"

~

1b.
What does "convertible databases" mean here, and will that term make
sense to a user? How are we "converting" the source server databases;
AFAIK we are simply connecting to them and adding publications.

IMO a better choice of adjectives can be found below.
"no suitable databases found..."
"no appropriate databases found..."
"no eligible databases found..."
etc.

Fixed.

The attached patch contains the required changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v8-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v8-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 905aa060191760fcb567467728085c1666d82bf3 Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v8] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 +++++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 124 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  98 ++++++++++++++
 3 files changed, 266 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e0..08f33479d41 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all-databases</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,26 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-databases</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       If a database is present on the source but missing on the target, an
+       error is raised.
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +135,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +255,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +272,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +288,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>--all-databases</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 2d881d54f5b..fe27c3dac62 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_databases;	/* --all-databases option was specified */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-databases             create subscriptions on the target for all source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1861,11 +1864,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all-databases is specified, fetch a list of all user-created databases
+ * from the source server. Internally, this is treated as if the user specified
+ * multiple --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_rows == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, false);
+		exit(1);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-databases", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1928,6 +1984,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all_databases = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1950,12 +2007,49 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				if (opt.all_databases)
+				{
+					pg_log_error("--all-databases specified multiple times");
+					exit(1);
+				}
+
+				if (num_dbs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --database");
+					exit(1);
+				}
+
+				if (num_pubs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --publication");
+					exit(1);
+				}
+
+				if (num_replslots > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --replication-slot");
+					exit(1);
+				}
+
+				if (num_subs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --subscription");
+					exit(1);
+				}
+				opt.all_databases = true;
+				break;
 			case 'd':
+				if (opt.all_databases)
+				{
+					pg_log_error("--database cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
 					simple_string_list_append(&opt.database_names, optarg);
@@ -1997,6 +2091,11 @@ main(int argc, char **argv)
 				opt.config_file = pg_strdup(optarg);
 				break;
 			case 2:
+				if (opt.all_databases)
+				{
+					pg_log_error("--publication cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.pub_names, optarg))
 				{
 					simple_string_list_append(&opt.pub_names, optarg);
@@ -2009,6 +2108,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 3:
+				if (opt.all_databases)
+				{
+					pg_log_error("--replication-slot cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.replslot_names, optarg))
 				{
 					simple_string_list_append(&opt.replslot_names, optarg);
@@ -2021,6 +2125,11 @@ main(int argc, char **argv)
 				}
 				break;
 			case 4:
+				if (opt.all_databases)
+				{
+					pg_log_error("--subscription cannot be used with --all-databases");
+					exit(1);
+				}
 				if (!simple_string_list_member(&opt.sub_names, optarg))
 				{
 					simple_string_list_append(&opt.sub_names, optarg);
@@ -2092,14 +2201,21 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all-databases is
+	 * specified.
+	 */
+	if (opt.all_databases)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all-databases option is provided, try
+		 * to obtain the dbname from the publisher conninfo. If dbname
+		 * parameter is not available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b7..33a6c03bf29 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,104 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with specifying '--all-databases' twice and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases', '--all-databases',
+	],
+	qr/--all-databases specified multiple times/,
+	'fail if --all-databases is used more than once');
+
+# run pg_createsubscriber with '--database' and '--all-databases' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --database/,
+	'fail if --all-databases is used with --database');
+
+# run pg_createsubscriber with '--publication' and '--all-databases' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all-databases/,
+	'fail if --all-databases is used with --publication');
+
+# run pg_createsubscriber with '--replication-slot' and '--all-databases' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all-databases',
+	],
+	qr/--all-databases cannot be used with --replication-slot/,
+	'fail if --all-databases is used with --replication-slot');
+
+# run pg_createsubscriber with '--subscription' and '--all-databases' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all-databases/,
+	'fail if --all-databases is used with --subscription');
+
+# run pg_createsubscriber with '--all-databases' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-databases',
+	],
+	'run pg_createsubscriber with --all-databases');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
-- 
2.41.0.windows.3

#35Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#34)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi Shubham,
Here are some review comments from my side

On Thu, Feb 13, 2025 at 8:58 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patch contains the required changes.

clusterdb, vacuumdb use -a and -all for all databases. Do we want to
use the same long option name here? Probably they don't have
-databases in the long option name since db is already in their name.
pg_createsubscriber doesn't have db in its name. But still it might be
better to have just -all to be consistent with those utilities. Sorry
if this has already been discussed.

+ printf(_(" -a, --all-databases create subscriptions on the target
for all source databases\n"));

Suggest "create subscriptions for all non-template source databases".
"on the target" seems unnecessary, it's implicit.

+
+# run pg_createsubscriber with '--database' and '--all-databases' and
verify the
+# failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all-databases',
+ ],
+ qr/--all-databases cannot be used with --database/,
+ 'fail if --all-databases is used with --database');

Need a test where --all-databases is specified before --database.

All the tests run in --dry-mode which won't test whether all the
replication slots, subscriptions and publications are working well
when --all-databases is specified. I think we need to test non-dry
mode creation. I may go as far as suggesting to split the current test
file into 3, 1. tests sanity of command line options 2. tests single
subscription creation thoroughly along with multiple --database
specifications 3. tests --all-databases health of the subscriptions
and --all-databases specific scenarios like non-template databases
aren't subscribed to.

--
Best Wishes,
Ashutosh Bapat

#36vignesh C
vignesh21@gmail.com
In reply to: vignesh C (#23)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, 10 Feb 2025 at 22:14, vignesh C <vignesh21@gmail.com> wrote:

On Mon, 10 Feb 2025 at 20:36, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patch contains the suggested changes.

If a new database is created on the primary server while
pg_createsubscriber is running, the subscription will not be created
for the new database.
To reproduce this issue, follow these steps:
Debug pg_createsubscriber and set a breakpoint after the
fetch_source_databases function, where the databases would have been
prepared.
While execution of pg_createsubscriber is paused, create a new
database on the primary node.
You will observe that the database is created on the standby node, but
the subscription will not be created for the newly created database.
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+       PGconn     *conn;
+       PGresult   *res;
+       int                     num_rows;
+
+       /* Establish a connection to the PostgreSQL server */
+       conn = connect_database(opt->pub_conninfo_str, true);
+
+       res = PQexec(conn, "SELECT datname FROM pg_database WHERE
datistemplate = false AND datallowconn = true");
+
+       /* Check for errors during query execution */
+       if (PQresultStatus(res) != PGRES_TUPLES_OK)

If we don't have a solution for this, how about the following approach:
a) Add a note in the documentation indicating that the publication and
subscriptions must be created manually for any new databases created
while pg_createsubscriber is running. b) At the end of the
pg_createsubscriber execution, identify any new databases that were
created and list them for the user.

Since this scenario might not be common, we could either implement
just option a) or both a) and b).

Regards,
Vignesh

#37Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: vignesh C (#36)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Vignesh,

If a new database is created on the primary server while
pg_createsubscriber is running, the subscription will not be created
for the new database.
To reproduce this issue, follow these steps:
Debug pg_createsubscriber and set a breakpoint after the
fetch_source_databases function, where the databases would have been
prepared.
While execution of pg_createsubscriber is paused, create a new
database on the primary node.
You will observe that the database is created on the standby node, but
the subscription will not be created for the newly created database.
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+       PGconn     *conn;
+       PGresult   *res;
+       int                     num_rows;
+
+       /* Establish a connection to the PostgreSQL server */
+       conn = connect_database(opt->pub_conninfo_str, true);
+
+       res = PQexec(conn, "SELECT datname FROM pg_database WHERE
datistemplate = false AND datallowconn = true");
+
+       /* Check for errors during query execution */
+       if (PQresultStatus(res) != PGRES_TUPLES_OK)

If we don't have a solution for this, how about the following approach:
a) Add a note in the documentation indicating that the publication and
subscriptions must be created manually for any new databases created
while pg_createsubscriber is running. b) At the end of the
pg_createsubscriber execution, identify any new databases that were
created and list them for the user.

Since this scenario might not be common, we could either implement
just option a) or both a) and b).

I'm not sure we must really handle the case. Documentation [1]https://www.postgresql.org/docs/devel/app-pgcreatesubscriber.html has already described
not to run DDL commands during the command:

```
Since DDL commands are not replicated by logical replication, avoid executing DDL
commands that change the database schema while running pg_createsubscriber.
If the target server has already been converted to logical replica, the DDL commands
might not be replicated, which might cause an error.
```

Thought?

[1]: https://www.postgresql.org/docs/devel/app-pgcreatesubscriber.html

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#38Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#34)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for updating the patch. Few comments.

```
+       If a database is present on the source but missing on the target, an
+       error is raised.
```

I'm not sure this description is accurate. I feel there is a case the command can
succeed.

Assuming that there are three databases (db1, db2, and db3) on the primary but db3
is missing on the standby when DBA is running the command. In check_subscriber(),
pg_createsubscriber connects to the db1 and it says OK. Then we wait until all
changes are replicated, thus db3 is created on the target. Everything may go well afterward.

Possible handling is as follows:

1) Ignore the corner case and retain,
2) Remove the description,
3) Modify the description, or
4) Modify codes to reject when the speficied database is missing on the target.

I feel 2) may be enough, but I want to ask others as well.

```
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
```

I do not think this is needed. If the database exists on the target but not on the
source, IIUC this means the DROP DATABASE was executed on the primary server but
not replicated to the standby yet. In this case such databases would be dropped
during the command because we ensure all changes are replicated before the promotion.
Tell me if there are other situations...

```
+               disconnect_database(conn, false);
+               exit(1);
```

This combination can be changed to disconnect_database(conn, true).

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#39Zhijie Hou (Fujitsu)
houzj.fnst@fujitsu.com
In reply to: Shubham Khanna (#34)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thursday, February 13, 2025 11:28 AM Shubham Khanna <khannashubham1197@gmail.com> wrote:

Hi,

The attached patch contains the required changes.

Thanks for updating the patch. I think it's a useful feature.

Here are few review comments:

1.

+				if (opt.all_databases)
+				{
+					pg_log_error("--all-databases specified multiple times");
+					exit(1);
+				}

The check for redundant --all-databases usage seems unnecessary as multiple
specifications do not cause harm or confusion for users. Similar server
commands with an --all option (such as clusterdb/vacuumdb/reindexdb) do not
report errors for it.

2.
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
...
+				if (num_dbs > 0)
+				{
+					pg_log_error("--all-databases cannot be used with --database");
+					exit(1);
+				}

I think the check for similar wrong combinations should not be done inside the
getopt_long() block. It should be performed after parsing to ensure all
cases are handled which would also be consistent with the methodology followed in
all other commands AFAICS.

Maybe we can write the error message in the format "%s cannot be used with %s"
to reduce the translation workload.

To be consistent with other error message for wrong option specification in this
command, consider adding pg_log_error_hint("Try \"%s --help\" for more information.").

3.

Ashutosh noted that commands like clusterdb and vacuumdb use the "--all" option
to indicate all databases, and I also found that the same is true for
pg_amcheck command, so I also think it's OK to use "-all" here.

Best Regards,
Hou zj

#40vignesh C
vignesh21@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#37)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, 18 Feb 2025 at 06:22, Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

I'm not sure we must really handle the case. Documentation [1] has already described
not to run DDL commands during the command:

```
Since DDL commands are not replicated by logical replication, avoid executing DDL
commands that change the database schema while running pg_createsubscriber.
If the target server has already been converted to logical replica, the DDL commands
might not be replicated, which might cause an error.
```

Yes, that makes sense. Since the same applies to tables created while
pg_createsubscriber is running, and there is already generic
documentation covering this, I agree with you.

Regards,
Vignesh

#41Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#35)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 14, 2025 at 5:27 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

Hi Shubham,
Here are some review comments from my side

On Thu, Feb 13, 2025 at 8:58 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patch contains the required changes.

clusterdb, vacuumdb use -a and -all for all databases. Do we want to
use the same long option name here? Probably they don't have
-databases in the long option name since db is already in their name.
pg_createsubscriber doesn't have db in its name. But still it might be
better to have just -all to be consistent with those utilities. Sorry
if this has already been discussed.

To maintain consistency with clusterdb and vacuumdb, I have changed
the option name from '--all-databases' to '--all'.

+ printf(_(" -a, --all-databases create subscriptions on the target
for all source databases\n"));

Suggest "create subscriptions for all non-template source databases".
"on the target" seems unnecessary, it's implicit.

Fixed.

+
+# run pg_createsubscriber with '--database' and '--all-databases' and
verify the
+# failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all-databases',
+ ],
+ qr/--all-databases cannot be used with --database/,
+ 'fail if --all-databases is used with --database');

Need a test where --all-databases is specified before --database.

Added a test case for this.

All the tests run in --dry-mode which won't test whether all the
replication slots, subscriptions and publications are working well
when --all-databases is specified. I think we need to test non-dry
mode creation. I may go as far as suggesting to split the current test
file into 3, 1. tests sanity of command line options 2. tests single
subscription creation thoroughly along with multiple --database
specifications 3. tests --all-databases health of the subscriptions
and --all-databases specific scenarios like non-template databases
aren't subscribed to.

--

I have added a test case for non-dry-run mode to ensure that
replication slots, subscriptions, and publications work as expected
when '--all' is specified. Additionally, I have split the test file
into two parts:
1) Command-line sanity checks – Validates the correctness of option parsing.
2) '--all' functionality and behavior – Verifies the health of
subscriptions and ensures that --all specific scenarios, such as
non-template databases not being subscribed to, are properly handled.
This should improve test coverage while keeping the structure manageable.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v9-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchapplication/octet-stream; name=v9-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patchDownload
From 097f92364d87768ed7ca5b47dbf80d78e5073777 Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v9] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the
'--all-databases' option. When '--all-databases' is specified, the tool
queries the source server (publisher) for all databases and creates
subscriptions on the target server (subscriber) for databases with matching
names. This simplifies the process of converting a physical standby to a
logical subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-databases' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  48 +++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 104 ++++++++++-
 .../t/040_pg_createsubscriber.pl              | 162 +++++++++++++++++-
 3 files changed, 305 insertions(+), 9 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d56487fe2c..4627c931e2 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +131,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +251,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +268,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +284,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 37fdf150b4..4d87bd0fd0 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all;			/* --all option was specified */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1879,11 +1882,62 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_rows == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1946,6 +2000,7 @@ main(int argc, char **argv)
 	opt.socket_dir = NULL;
 	opt.sub_port = DEFAULT_SUB_PORT;
 	opt.sub_username = NULL;
+	opt.all = false;
 	opt.database_names = (SimpleStringList)
 	{
 		0
@@ -1968,11 +2023,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2057,6 +2115,38 @@ main(int argc, char **argv)
 		}
 	}
 
+	if (opt.all)
+	{
+		if (num_dbs > 0)
+		{
+			pg_log_error("%s cannot be used with %s", "--all", "--database");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		if (num_pubs > 0)
+		{
+			pg_log_error("%s cannot be used with %s", "--all", "--publication");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		if (num_replslots > 0)
+		{
+			pg_log_error("%s cannot be used with %s", "--all", "--replication-slot");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+
+		if (num_subs > 0)
+		{
+			pg_log_error("%s cannot be used with %s", "--all", "--subscription");
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2110,14 +2200,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..837a2f1f86 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,165 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--all cannot be used with --database/,
+	'fail if --all is used with --database');
+
+# run pg_createsubscriber with '--database' and '--all' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--all cannot be used with --database/,
+	'fail if --all is used with --database');
+
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--all cannot be used with --database/,
+	'fail if --all is used with --database');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--all cannot be used with --publication/,
+	'fail if --all is used with --publication');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--all cannot be used with --replication-slot/,
+	'fail if --all is used with --replication-slot');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--all cannot be used with --subscription/,
+	'fail if --all is used with --subscription');
+
+# run pg_createsubscriber with '--all' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Set up node S1 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s1 = PostgreSQL::Test::Cluster->new('node_s1');
+$node_s1->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_s1->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s1->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_s1->start;
+
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+	my $sub_exists =
+	  $node_s1->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;");
+	is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
+
+# Verify replication is working
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES ('replication test');");
+
+$result = $node_s1->safe_psql($db1, "SELECT * FROM tbl1 ORDER BY 1");
+is( $result, qq(first row
+replication test
+second row), "replication successful in $db1");
+
+$node_s1->stop;
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
@@ -431,8 +590,9 @@ $result = $node_s->safe_psql($db1,
 is($result, qq(0), 'failover slot was removed');
 
 # Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1 ORDER BY 1');
 is( $result, qq(first row
+replication test
 second row
 third row),
 	"logical replication works in database $db1");
-- 
2.34.1

#42Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#38)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Feb 18, 2025 at 7:32 AM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch. Few comments.

```
+       If a database is present on the source but missing on the target, an
+       error is raised.
```

I'm not sure this description is accurate. I feel there is a case the command can
succeed.

Assuming that there are three databases (db1, db2, and db3) on the primary but db3
is missing on the standby when DBA is running the command. In check_subscriber(),
pg_createsubscriber connects to the db1 and it says OK. Then we wait until all
changes are replicated, thus db3 is created on the target. Everything may go well afterward.

Possible handling is as follows:

1) Ignore the corner case and retain,
2) Remove the description,
3) Modify the description, or
4) Modify codes to reject when the speficied database is missing on the target.

I feel 2) may be enough, but I want to ask others as well.

I agree with you on this. Since there are cases where the command can
still succeed even if a database is initially missing on the target,
the description could be misleading. Based on this, I have removed
that part of the description.

```
+       If a database exists on the target but not on the source, no
+       subscription is created for it.
```

I do not think this is needed. If the database exists on the target but not on the
source, IIUC this means the DROP DATABASE was executed on the primary server but
not replicated to the standby yet. In this case such databases would be dropped
during the command because we ensure all changes are replicated before the promotion.
Tell me if there are other situations...

I agree with your reasoning. Since any databases that exist on the
target but not on the source would be dropped as part of ensuring all
changes are replicated before promotion, explicitly mentioning this
scenario isn’t necessary. Based on this, I have removed that part of
the description.

```
+               disconnect_database(conn, false);
+               exit(1);
```

This combination can be changed to disconnect_database(conn, true).

Fixed.

The attached patch at [1]/messages/by-id/CAHv8RjKAdrrt3-pF6yHb5gBricB9=D7O47Dxe39zRxKkShdpmw@mail.gmail.com contains the suggested changes.
[1]: /messages/by-id/CAHv8RjKAdrrt3-pF6yHb5gBricB9=D7O47Dxe39zRxKkShdpmw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#43Shubham Khanna
khannashubham1197@gmail.com
In reply to: Zhijie Hou (Fujitsu) (#39)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Feb 18, 2025 at 2:43 PM Zhijie Hou (Fujitsu)
<houzj.fnst@fujitsu.com> wrote:

On Thursday, February 13, 2025 11:28 AM Shubham Khanna <khannashubham1197@gmail.com> wrote:

Hi,

The attached patch contains the required changes.

Thanks for updating the patch. I think it's a useful feature.

Here are few review comments:

1.

+                               if (opt.all_databases)
+                               {
+                                       pg_log_error("--all-databases specified multiple times");
+                                       exit(1);
+                               }

The check for redundant --all-databases usage seems unnecessary as multiple
specifications do not cause harm or confusion for users. Similar server
commands with an --all option (such as clusterdb/vacuumdb/reindexdb) do not
report errors for it.

I agree with your point. Since multiple specifications of '--all' do
not cause any issues or confusion, and other similar commands (like
clusterdb, vacuumdb, and reindexdb) do not enforce this restriction, I
have removed the test case accordingly.

2.
+       while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
long_options, &option_index)) != -1)
...
+                               if (num_dbs > 0)
+                               {
+                                       pg_log_error("--all-databases cannot be used with --database");
+                                       exit(1);
+                               }

I think the check for similar wrong combinations should not be done inside the
getopt_long() block. It should be performed after parsing to ensure all
cases are handled which would also be consistent with the methodology followed in
all other commands AFAICS.

I agree with your suggestion. To ensure consistency with other
commands, I have moved the check for conflicting options outside the
getopt_long() block. This allows us to handle all cases after parsing
is complete.

Maybe we can write the error message in the format "%s cannot be used with %s"
to reduce the translation workload.

Fixed.

To be consistent with other error message for wrong option specification in this
command, consider adding pg_log_error_hint("Try \"%s --help\" for more information.").

Fixed.

3.

Ashutosh noted that commands like clusterdb and vacuumdb use the "--all" option
to indicate all databases, and I also found that the same is true for
pg_amcheck command, so I also think it's OK to use "-all" here.

Fixed.

The attached patch at [1]/messages/by-id/CAHv8RjKAdrrt3-pF6yHb5gBricB9=D7O47Dxe39zRxKkShdpmw@mail.gmail.com contains the suggested changes.
[1]: /messages/by-id/CAHv8RjKAdrrt3-pF6yHb5gBricB9=D7O47Dxe39zRxKkShdpmw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#44Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#41)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for updating the patch!

I have added a test case for non-dry-run mode to ensure that
replication slots, subscriptions, and publications work as expected
when '--all' is specified. Additionally, I have split the test file
into two parts:
1) Command-line sanity checks – Validates the correctness of option parsing.
2) '--all' functionality and behavior – Verifies the health of
subscriptions and ensures that --all specific scenarios, such as
non-template databases not being subscribed to, are properly handled.
This should improve test coverage while keeping the structure manageable.

TBH, I feel your change does not separate the test file into the two parts. ISTM
you just added some validation checks and a test how --all option works.
Ashutosh, does it match your suggestion?

Anyway, here are my comments.

01.
```
+ int num_rows;
```

I think num_rows is not needed, we can do like:

```
        /* Process the query result */
-       num_rows = PQntuples(res);
-       for (int i = 0; i < num_rows; i++)
+       for (int i = 0; i < PQntuples(res); i+
```
And
```
        /* Error if no databases were found on the source server */
-       if (num_rows == 0)
+       if (num_dbs == 0)
```

02.
```
+ opt.all = false;
```

This must be done after initializing recovery_timeout.

03.
```
+# Set up node S1 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s1 = PostgreSQL::Test::Cluster->new('node_s1');
```

I do not like the name "S1" because this is the second standby sever
from the node_p.

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+	my $sub_exists =
+	  $node_s1->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;");
+	is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so that
very loop will see exactly the same content. Also, 'postgres' is also the user database.
I feel you must ensure that all three databases (postgres, $db1, and $db2) have a
subscription here.

05.
```
+# Verify replication is working
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES ('replication test');");
+
+$result = $node_s1->safe_psql($db1, "SELECT * FROM tbl1 ORDER BY 1");
+is( $result, qq(first row
+replication test
+second row), "replication successful in $db1");
+
+$node_s1->stop;
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
```

I'm not sure the part is not needed. This have already been checked in other parts, right?

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#45Peter Smith
smithpb2250@gmail.com
In reply to: Shubham Khanna (#41)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi Shubham, Here are some review comments for v9-0001.

======
Commit message

1.
You've changed the option to '--all' but the comment message still
refers to '-all-databases'

======
doc/src/sgml/ref/pg_createsubscriber.sgml

2.
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.

I don't see what is "automatic" about this, nor do I know really what
"fetch" of databases is trying to convey.

Isn't it simpler to just say like below?

SUGGESTION
For all source server non-template databases create subscriptions for...

======
src/bin/pg_basebackup/pg_createsubscriber.c

3.
int recovery_timeout; /* stop recovery after this time */
+ bool all; /* --all option was specified */
};

IMO 'all' is too generic for a field name; it has almost no meaning --
all what? I think the original 'all_databases' name was better. Or
'all_dbs', but not just 'all'.

~~~

4.
+ if (opt.all)
+ {
+ if (num_dbs > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--database");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }
+
+ if (num_pubs > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--publication");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }
+
+ if (num_replslots > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--replication-slot");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }
+
+ if (num_subs > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--subscription");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }

4a
If bad combinations exist then IMO it is most likely that the --all
came before the incompatible option. So the parameters should be the
other way around so the result is "--<XXX> cannot be used with --all".

~

4b.
You could eliminate all this code duplication by using a variable for
the bad option,

SUGGESTION
char *bad_switch = NULL;

if (num_dbs > 0) bad_switch = "--database";
else if (num_pubs > 0) bad_switch = "--publication";
else if (num_replslots > 0) bad_switch = "--replication_slot";
else if (num_subs > 0) bad_switch = "--subscription";

if (bad_switch)
{
pg_log_error("%s cannot be used with --all", bad_switch);
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
exit(1);
}

~

4c.
There is a double-blank line after this code fragment.

======
.../t/040_pg_createsubscriber.pl

5.
+# run pg_createsubscriber with '--database' and '--all' and verify the
+# failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--all cannot be used with --database/,
+ 'fail if --all is used with --database');
+

There is no difference anymore in the logic/error if the options are
given in order "--all --database" or if they are in order "--database
--all". So you no longer need to have separate test cases for
different ordering.

~~~

6.
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+ my $sub_exists =
+   $node_s1->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;");
+ is($sub_exists, '3', "Subscription created successfully for $dbname");
+}

AFAICT this is just checking a global subscription count in a loop so
it will be 3, 3, 3. Why?

I guess you intended to verify that relevant subscriptions were
created for each of the target databases. But this code is not doing
that.

~~~

7.
+# Verify replication is working
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES ('replication test');");
+
+$result = $node_s1->safe_psql($db1, "SELECT * FROM tbl1 ORDER BY 1");
+is( $result, qq(first row
+replication test
+second row), "replication successful in $db1");
+
+$node_s1->stop;
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
@@ -431,8 +590,9 @@ $result = $node_s->safe_psql($db1,
 is($result, qq(0), 'failover slot was removed');
 # Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1 ORDER BY 1');
 is( $result, qq(first row
+replication test

I concur with Kuroda-san's comment. Maybe remove all this, because
AFAICT existing code was already testing that replication is working
ok for $db1 and $db2.

======
Kind Regards,
Peter Smith.
Fujitsu Australia

#46Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#44)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 19, 2025 at 4:23 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch!

I have added a test case for non-dry-run mode to ensure that
replication slots, subscriptions, and publications work as expected
when '--all' is specified. Additionally, I have split the test file
into two parts:
1) Command-line sanity checks – Validates the correctness of option parsing.
2) '--all' functionality and behavior – Verifies the health of
subscriptions and ensures that --all specific scenarios, such as
non-template databases not being subscribed to, are properly handled.
This should improve test coverage while keeping the structure manageable.

TBH, I feel your change does not separate the test file into the two parts. ISTM
you just added some validation checks and a test how --all option works.
Ashutosh, does it match your suggestion?

Anyway, here are my comments.

01.
```
+ int num_rows;
```

I think num_rows is not needed, we can do like:

```
/* Process the query result */
-       num_rows = PQntuples(res);
-       for (int i = 0; i < num_rows; i++)
+       for (int i = 0; i < PQntuples(res); i+
```
And
```
/* Error if no databases were found on the source server */
-       if (num_rows == 0)
+       if (num_dbs == 0)
```

Fixed.

02.
```
+ opt.all = false;
```

This must be done after initializing recovery_timeout.

Fixed.

03.
```
+# Set up node S1 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s1 = PostgreSQL::Test::Cluster->new('node_s1');
```

I do not like the name "S1" because this is the second standby sever
from the node_p.

Fixed.

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;");
+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so that
very loop will see exactly the same content. Also, 'postgres' is also the user database.
I feel you must ensure that all three databases (postgres, $db1, and $db2) have a
subscription here.

Fixed.

05.
```
+# Verify replication is working
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES ('replication test');");
+
+$result = $node_s1->safe_psql($db1, "SELECT * FROM tbl1 ORDER BY 1");
+is( $result, qq(first row
+replication test
+second row), "replication successful in $db1");
+
+$node_s1->stop;
# Run pg_createsubscriber on node S.  --verbose is used twice
# to show more information.
command_ok(
```

I'm not sure the part is not needed. This have already been checked in other parts, right?

Fixed.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v10-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v10-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From a30bb5cc1f07ae31c00abe97e9a212bbc0d8999b Mon Sep 17 00:00:00 2001
From: Shubham Khanna <shubham.khanna@fujitsu.com>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v10] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all-dbs'
option. When '--all-dbs' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all-dbs' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  48 ++++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 +++++++++++-
 .../t/040_pg_createsubscriber.pl              | 135 +++++++++++++++++-
 3 files changed, 266 insertions(+), 9 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d56487fe2c..d87c814b25 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all-dbs</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,7 +131,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. Cannot be used together with <option>--all-dbs</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -214,7 +251,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>--all-dbs</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -230,7 +268,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all-dbs</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -245,7 +284,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>--all-dbs</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 9fdf15e5ac..afe9f5cc81 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -43,6 +43,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 struct LogicalRepInfo
@@ -106,6 +107,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -220,6 +222,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all-dbs                   create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1883,11 +1886,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all-dbs", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1955,6 +2007,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1972,11 +2025,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:U:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:U:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2061,6 +2117,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all-dbs is not used with incompatible options. */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all-dbs", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2114,14 +2192,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..e3cdf1ea7d 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,139 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all-dbs' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-dbs',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all-dbs/,
+	'fail if --database is used with --all-dbs');
+
+# run pg_createsubscriber with '--database' and '--all-dbs' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all-dbs',
+	],
+	qr/--database cannot be used with --all-dbs/,
+	'fail if --database is used with --all-dbs');
+
+# run pg_createsubscriber with '--publication' and '--all-dbs' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-dbs',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all-dbs/,
+	'fail if --publication is used with --all-dbs');
+
+# run pg_createsubscriber with '--replication-slot' and '--all-dbs' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all-dbs',
+	],
+	qr/--replication-slot cannot be used with --all-dbs/,
+	'fail if --replication-slot is used with --all-dbs');
+
+# run pg_createsubscriber with '--subscription' and '--all-dbs' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-dbs',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all-dbs/,
+	'fail if --subscription is used with --all-dbs');
+
+# run pg_createsubscriber with '--all-dbs' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all-dbs',
+	],
+	'run pg_createsubscriber with --all-dbs');
+
+# Set up node S2 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s2 = PostgreSQL::Test::Cluster->new('node_s2');
+$node_s2->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_s2->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s2->set_standby_mode();
+
+# run pg_createsubscriber with '--all-dbs' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s2->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s2->host,
+		'--subscriber-port' => $node_s2->port,
+		'--all-dbs',
+	],
+	'run pg_createsubscriber with --all-dbs');
+
+$node_s2->start;
+
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ('postgres', $db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+	my $sub_exists =
+	  $node_s2->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;");
+	is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 command_ok(
@@ -431,7 +564,7 @@ $result = $node_s->safe_psql($db1,
 is($result, qq(0), 'failover slot was removed');
 
 # Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1 ORDER BY 1');
 is( $result, qq(first row
 second row
 third row),
-- 
2.34.1

#47Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#45)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Feb 20, 2025 at 1:23 PM Peter Smith <smithpb2250@gmail.com> wrote:

Hi Shubham, Here are some review comments for v9-0001.

======
Commit message

1.
You've changed the option to '--all' but the comment message still
refers to '-all-databases'

Fixed.

======
doc/src/sgml/ref/pg_createsubscriber.sgml

2.
+      <para>
+       Automatically fetch all non-template databases from the source server
+       and create subscriptions for databases with the same names on the
+       target server.

I don't see what is "automatic" about this, nor do I know really what
"fetch" of databases is trying to convey.

Isn't it simpler to just say like below?

SUGGESTION
For all source server non-template databases create subscriptions for...

Fixed.

======
src/bin/pg_basebackup/pg_createsubscriber.c

3.
int recovery_timeout; /* stop recovery after this time */
+ bool all; /* --all option was specified */
};

IMO 'all' is too generic for a field name; it has almost no meaning --
all what? I think the original 'all_databases' name was better. Or
'all_dbs', but not just 'all'.

~~~

Fixed.

4.
+ if (opt.all)
+ {
+ if (num_dbs > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--database");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }
+
+ if (num_pubs > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--publication");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }
+
+ if (num_replslots > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--replication-slot");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }
+
+ if (num_subs > 0)
+ {
+ pg_log_error("%s cannot be used with %s", "--all", "--subscription");
+ pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+ exit(1);
+ }

4a
If bad combinations exist then IMO it is most likely that the --all
came before the incompatible option. So the parameters should be the
other way around so the result is "--<XXX> cannot be used with --all".

~

Fixed.

4b.
You could eliminate all this code duplication by using a variable for
the bad option,

SUGGESTION
char *bad_switch = NULL;

if (num_dbs > 0) bad_switch = "--database";
else if (num_pubs > 0) bad_switch = "--publication";
else if (num_replslots > 0) bad_switch = "--replication_slot";
else if (num_subs > 0) bad_switch = "--subscription";

if (bad_switch)
{
pg_log_error("%s cannot be used with --all", bad_switch);
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
exit(1);
}

~

Fixed.

4c.
There is a double-blank line after this code fragment.

Fixed.

======
.../t/040_pg_createsubscriber.pl

5.
+# run pg_createsubscriber with '--database' and '--all' and verify the
+# failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--all cannot be used with --database/,
+ 'fail if --all is used with --database');
+

There is no difference anymore in the logic/error if the options are
given in order "--all --database" or if they are in order "--database
--all". So you no longer need to have separate test cases for
different ordering.

~~~

Fixed.

6.
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+ my $sub_exists =
+   $node_s1->safe_psql($dbname, "SELECT count(*) FROM pg_subscription;");
+ is($sub_exists, '3', "Subscription created successfully for $dbname");
+}

AFAICT this is just checking a global subscription count in a loop so
it will be 3, 3, 3. Why?

I guess you intended to verify that relevant subscriptions were
created for each of the target databases. But this code is not doing
that.

~~~

Fixed.

7.
+# Verify replication is working
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES ('replication test');");
+
+$result = $node_s1->safe_psql($db1, "SELECT * FROM tbl1 ORDER BY 1");
+is( $result, qq(first row
+replication test
+second row), "replication successful in $db1");
+
+$node_s1->stop;
# Run pg_createsubscriber on node S.  --verbose is used twice
# to show more information.
command_ok(
@@ -431,8 +590,9 @@ $result = $node_s->safe_psql($db1,
is($result, qq(0), 'failover slot was removed');
# Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1 ORDER BY 1');
is( $result, qq(first row
+replication test

I concur with Kuroda-san's comment. Maybe remove all this, because
AFAICT existing code was already testing that replication is working
ok for $db1 and $db2.

======

Fixed.

The attached patch at [1]/messages/by-id/CAHv8RjLtgrA9odXtwhit1mUfqogNSF4qhkvDrPbxEoWba+4SOw@mail.gmail.com contains the suggested changes.

[1]: /messages/by-id/CAHv8RjLtgrA9odXtwhit1mUfqogNSF4qhkvDrPbxEoWba+4SOw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#48Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#46)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for updating the patch quickly!

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM

pg_subscription;");

+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so

that

very loop will see exactly the same content. Also, 'postgres' is also the user

database.

I feel you must ensure that all three databases (postgres, $db1, and $db2) have

a

subscription here.

Fixed.

My point was that the loop does not have meaning because pg_subscription
is a global one. I and Peter considered changes like [1]``` # Verify that only user databases got subscriptions (not template databases) my @user_dbs = ('postgres', $db1, $db2); foreach my $dbname (@user_dbs) { $result = $node_s2->safe_psql('postgres', "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datname = '$dbname';"); is needed. It ensures
that each databases have a subscription.
Note: [1]``` # Verify that only user databases got subscriptions (not template databases) my @user_dbs = ('postgres', $db1, $db2); foreach my $dbname (@user_dbs) { $result = $node_s2->safe_psql('postgres', "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datname = '$dbname';"); cannot pass the test as-is because $db1 and $db2 contains special
characters. Please escape appropriately.

Other comments are listed in below.

01.
```
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
```

Peter's comment [2]/messages/by-id/CAHut+PsatTfk9-F4JBrX_yYE0QGh4wQiTmvS4=dnBxcL=AK2HA@mail.gmail.com does not say that option name should be changed.
The scope of his comment is only in the .c file.

02.
```
+# Set up node S2 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s2 = PostgreSQL::Test::Cluster->new('node_s2');
```

I feel $node_s should be renamed to $node_s1, then you can use $node_s2.

Note that this change may not be needed based on the comment [3]/messages/by-id/CAExHW5vmMs5nZ6=XcCYAXMJrhVrsW7hOovyg+P+T9Pkuc7DykA@mail.gmail.com.
We may have to separate the test file.

[1]: ``` # Verify that only user databases got subscriptions (not template databases) my @user_dbs = ('postgres', $db1, $db2); foreach my $dbname (@user_dbs) { $result = $node_s2->safe_psql('postgres', "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datname = '$dbname';");
```
# Verify that only user databases got subscriptions (not template databases)
my @user_dbs = ('postgres', $db1, $db2);
foreach my $dbname (@user_dbs)
{
$result =
$node_s2->safe_psql('postgres',
"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datname = '$dbname';");

is($result, '1', "Subscription created successfully for $dbname");
}
```
[2]: /messages/by-id/CAHut+PsatTfk9-F4JBrX_yYE0QGh4wQiTmvS4=dnBxcL=AK2HA@mail.gmail.com
[3]: /messages/by-id/CAExHW5vmMs5nZ6=XcCYAXMJrhVrsW7hOovyg+P+T9Pkuc7DykA@mail.gmail.com

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#49Peter Smith
smithpb2250@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#48)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Feb 20, 2025 at 10:26 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch quickly!

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM

pg_subscription;");

+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so

that

very loop will see exactly the same content. Also, 'postgres' is also the user

database.

I feel you must ensure that all three databases (postgres, $db1, and $db2) have

a

subscription here.

Fixed.

My point was that the loop does not have meaning because pg_subscription
is a global one. I and Peter considered changes like [1] is needed. It ensures
that each databases have a subscription.
Note: [1] cannot pass the test as-is because $db1 and $db2 contains special
characters. Please escape appropriately.

Yes. Some test is still needed to confirm the expected subscriptions
all get created for respective dbs. But, the current test loop just
isn't doing it properly.

Other comments are listed in below.

01.
```
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
```

Peter's comment [2] does not say that option name should be changed.
The scope of his comment is only in the .c file.

Yes, that's correct. My v9 suggestion to change 'all' to 'all_dbs' was
referring only to the field name of struct CreateSubscriberOptions,
nothing else. Not the usage help, not the error messages, not the
docs, not the tests, not the commit message.

======
Kind Regards,
Peter Smith.
Fujitsu Australia

#50Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Peter Smith (#49)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 21, 2025 at 5:18 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Thu, Feb 20, 2025 at 10:26 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch quickly!

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM

pg_subscription;");

+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so

that

very loop will see exactly the same content. Also, 'postgres' is also the user

database.

I feel you must ensure that all three databases (postgres, $db1, and $db2) have

a

subscription here.

Fixed.

My point was that the loop does not have meaning because pg_subscription
is a global one. I and Peter considered changes like [1] is needed. It ensures
that each databases have a subscription.
Note: [1] cannot pass the test as-is because $db1 and $db2 contains special
characters. Please escape appropriately.

Yes. Some test is still needed to confirm the expected subscriptions
all get created for respective dbs. But, the current test loop just
isn't doing it properly.

Other comments are listed in below.

01.
```
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
```

Peter's comment [2] does not say that option name should be changed.
The scope of his comment is only in the .c file.

Yes, that's correct. My v9 suggestion to change 'all' to 'all_dbs' was
referring only to the field name of struct CreateSubscriberOptions,
nothing else. Not the usage help, not the error messages, not the
docs, not the tests, not the commit message.

+1. We don't want yet another option to specify all databases. :)

--
Best Wishes,
Ashutosh Bapat

#51Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#44)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Feb 19, 2025 at 4:23 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch!

I have added a test case for non-dry-run mode to ensure that
replication slots, subscriptions, and publications work as expected
when '--all' is specified. Additionally, I have split the test file
into two parts:
1) Command-line sanity checks – Validates the correctness of option parsing.
2) '--all' functionality and behavior – Verifies the health of
subscriptions and ensures that --all specific scenarios, such as
non-template databases not being subscribed to, are properly handled.
This should improve test coverage while keeping the structure manageable.

TBH, I feel your change does not separate the test file into the two parts. ISTM
you just added some validation checks and a test how --all option works.
Ashutosh, does it match your suggestion?

Thanks Hayato for pointing it out. The test changes don't match my
expectations. As you rightly pointed out, I expected two (actually
three if needed) separate test files one for argument validation and
one for testing --database scenarios (the existing tests) and one more
for testing same scenarios when --all is specified. Right now all it
does is "# Verify that only user databases got subscriptions (not
template databases)". I expected testing the actual replication as
well like tests between lines around 527 and 582 in the latest
patchset. Those tests are performed when --database is subscribed. We
need similar tests performed when --all is specified. I didn't find
any of those tests being performed on node_s2. Given that the tests
for --databases and --all will be very similar, having them in the
same test file makes more sense. We also seem to be missing
$node_s2->teardown_node, do we?

--
Best Wishes,
Ashutosh Bapat

#52Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#48)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Feb 20, 2025 at 4:56 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch quickly!

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM

pg_subscription;");

+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so

that

very loop will see exactly the same content. Also, 'postgres' is also the user

database.

I feel you must ensure that all three databases (postgres, $db1, and $db2) have

a

subscription here.

Fixed.

My point was that the loop does not have meaning because pg_subscription
is a global one. I and Peter considered changes like [1] is needed. It ensures
that each databases have a subscription.
Note: [1] cannot pass the test as-is because $db1 and $db2 contains special
characters. Please escape appropriately.

Fixed.

Other comments are listed in below.

01.
```
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
```

Peter's comment [2] does not say that option name should be changed.
The scope of his comment is only in the .c file.

Fixed.

02.
```
+# Set up node S2 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s2 = PostgreSQL::Test::Cluster->new('node_s2');
```

I feel $node_s should be renamed to $node_s1, then you can use $node_s2.

Note that this change may not be needed based on the comment [3].

Fixed.

We may have to separate the test file.

[1]:
```
# Verify that only user databases got subscriptions (not template databases)
my @user_dbs = ('postgres', $db1, $db2);
foreach my $dbname (@user_dbs)
{
$result =
$node_s2->safe_psql('postgres',
"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datname = '$dbname';");

is($result, '1', "Subscription created successfully for $dbname");
}
```

Fixed.

[2]: /messages/by-id/CAHut+PsatTfk9-F4JBrX_yYE0QGh4wQiTmvS4=dnBxcL=AK2HA@mail.gmail.com
[3]: /messages/by-id/CAExHW5vmMs5nZ6=XcCYAXMJrhVrsW7hOovyg+P+T9Pkuc7DykA@mail.gmail.com

The attached Patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v11-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v11-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From af7f7d508cbb2d04bfe4100621629627ad89d37c Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 28 Feb 2025 17:29:30 +0530
Subject: [PATCH v11] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  53 +++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 ++++++-
 .../t/040_pg_createsubscriber.pl              | 242 ++++++++++++++----
 3 files changed, 326 insertions(+), 61 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index b4b996236e4..0878effb442 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,9 +131,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. Cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.
       </para>
      </listitem>
@@ -230,7 +268,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. Cannot be used
+       together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -246,7 +285,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. Cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -261,7 +301,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. Cannot be used
+       together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index a5a2d61165d..d2771a81780 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -44,6 +44,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 /* per-database publication/subscription info */
@@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -234,6 +236,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1902,11 +1905,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1976,6 +2028,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1993,11 +2046,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2085,6 +2141,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options. */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2138,14 +2216,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c35fa108ce3..7574a3903e9 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -152,16 +152,16 @@ $node_p->safe_psql($db2,
 
 # Set up node S as standby linking to node P
 $node_p->backup('backup_1');
-my $node_s = PostgreSQL::Test::Cluster->new('node_s');
-$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
-$node_s->append_conf(
+my $node_s1 = PostgreSQL::Test::Cluster->new('node_s1');
+$node_s1->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_s1->append_conf(
 	'postgresql.conf', qq[
 primary_slot_name = '$slotname'
 primary_conninfo = '$pconnstr dbname=postgres'
 hot_standby_feedback = on
 ]);
-$node_s->set_standby_mode();
-$node_s->start;
+$node_s1->set_standby_mode();
+$node_s1->start;
 
 # Set up node T as standby linking to node P then promote it
 my $node_t = PostgreSQL::Test::Cluster->new('node_t');
@@ -192,10 +192,10 @@ command_fails(
 		'pg_createsubscriber',
 		'--verbose',
 		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
+		'--pgdata' => $node_s1->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
 		'--database' => $db1,
 		'--database' => $db2,
 	],
@@ -216,9 +216,9 @@ command_fails(
 	'subscriber data directory is not a copy of the source database cluster');
 
 # Set up node C as standby linking to node S
-$node_s->backup('backup_2');
+$node_s1->backup('backup_2');
 my $node_c = PostgreSQL::Test::Cluster->new('node_c');
-$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1);
+$node_c->init_from_backup($node_s1, 'backup_2', has_streaming => 1);
 $node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef);
 $node_c->set_standby_mode();
 
@@ -229,7 +229,7 @@ command_fails(
 		'--verbose',
 		'--dry-run',
 		'--pgdata' => $node_c->data_dir,
-		'--publisher-server' => $node_s->connstr($db1),
+		'--publisher-server' => $node_s1->connstr($db1),
 		'--socketdir' => $node_c->host,
 		'--subscriber-port' => $node_c->port,
 		'--database' => $db1,
@@ -246,16 +246,16 @@ max_wal_senders = 1
 max_worker_processes = 2
 });
 $node_p->restart;
-$node_s->stop;
+$node_s1->stop;
 command_fails(
 	[
 		'pg_createsubscriber',
 		'--verbose',
 		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
+		'--pgdata' => $node_s1->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
 		'--database' => $db1,
 		'--database' => $db2,
 
@@ -272,7 +272,7 @@ max_worker_processes = 8
 });
 
 # Check some unmet conditions on node S
-$node_s->append_conf(
+$node_s1->append_conf(
 	'postgresql.conf', q{
 max_replication_slots = 1
 max_logical_replication_workers = 1
@@ -283,15 +283,15 @@ command_fails(
 		'pg_createsubscriber',
 		'--verbose',
 		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
+		'--pgdata' => $node_s1->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
 		'--database' => $db1,
 		'--database' => $db2,
 	],
 	'standby contains unmet conditions on node S');
-$node_s->append_conf(
+$node_s1->append_conf(
 	'postgresql.conf', q{
 max_replication_slots = 10
 max_logical_replication_workers = 4
@@ -305,12 +305,12 @@ my $fslotname = 'failover_slot';
 $node_p->safe_psql($db1,
 	"SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)"
 );
-$node_s->start;
+$node_s1->start;
 # Wait for the standby to catch up so that the standby is not lagging behind
 # the failover slot.
-$node_p->wait_for_replay_catchup($node_s);
-$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()");
-my $result = $node_s->safe_psql('postgres',
+$node_p->wait_for_replay_catchup($node_s1);
+$node_s1->safe_psql('postgres', "SELECT pg_sync_replication_slots()");
+my $result = $node_s1->safe_psql('postgres',
 	"SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary"
 );
 is($result, 'failover_slot', 'failover slot is synced');
@@ -321,15 +321,15 @@ is($result, 'failover_slot', 'failover slot is synced');
 # slot) xmin on standby could be ahead of the remote slot leading
 # to failure in synchronization.
 $node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')");
-$node_p->wait_for_replay_catchup($node_s);
+$node_p->wait_for_replay_catchup($node_s1);
 
 # Create subscription to test its removal
 my $dummy_sub = 'regress_sub_dummy';
 $node_p->safe_psql($db1,
 	"CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)"
 );
-$node_p->wait_for_replay_catchup($node_s);
-$node_s->stop;
+$node_p->wait_for_replay_catchup($node_s1);
+$node_s1->stop;
 
 # dry run mode on node S
 command_ok(
@@ -338,10 +338,10 @@ command_ok(
 		'--verbose',
 		'--dry-run',
 		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
-		'--pgdata' => $node_s->data_dir,
+		'--pgdata' => $node_s1->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
 		'--publication' => 'pub1',
 		'--publication' => 'pub2',
 		'--subscription' => 'sub1',
@@ -352,10 +352,11 @@ command_ok(
 	'run pg_createsubscriber --dry-run on node S');
 
 # Check if node S is still a standby
-$node_s->start;
-is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'),
-	't', 'standby is in recovery');
-$node_s->stop;
+$node_s1->start;
+is( $node_s1->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'),
+	't',
+	'standby is in recovery');
+$node_s1->stop;
 
 # pg_createsubscriber can run without --databases option
 command_ok(
@@ -363,14 +364,152 @@ command_ok(
 		'pg_createsubscriber',
 		'--verbose',
 		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
+		'--pgdata' => $node_s1->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
 		'--replication-slot' => 'replslot1',
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
+# run pg_createsubscriber with '--all' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s1->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Set up node S2 as standby linking to node P
+$node_p->backup('backup_3');
+my $node_s2 = PostgreSQL::Test::Cluster->new('node_s2');
+$node_s2->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_s2->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s2->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s2->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s2->host,
+		'--subscriber-port' => $node_s2->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_s2->start;
+
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ('postgres', $db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+	$result = $node_s2->safe_psql('postgres',
+		"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f';"
+	);
+	is($result, '3', "Subscription created successfully for $dbname");
+	$result = $node_s2->safe_psql('postgres',
+		"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
+	);
+	is($result, '0', "Subscription created successfully for $dbname");
+}
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option
@@ -379,10 +518,10 @@ command_ok(
 		'pg_createsubscriber',
 		'--verbose', '--verbose',
 		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
-		'--pgdata' => $node_s->data_dir,
+		'--pgdata' => $node_s1->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
+		'--socketdir' => $node_s1->host,
+		'--subscriber-port' => $node_s1->port,
 		'--publication' => 'pub1',
 		'--publication' => 'pub2',
 		'--replication-slot' => 'replslot1',
@@ -406,11 +545,11 @@ $node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')");
 $node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')");
 
 # Start subscriber
-$node_s->start;
+$node_s1->start;
 
 # Verify that all subtwophase states are pending or enabled,
 # e.g. there are no subscriptions where subtwophase is disabled ('d')
-is( $node_s->safe_psql(
+is( $node_s1->safe_psql(
 		'postgres',
 		"SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'"
 	),
@@ -418,50 +557,51 @@ is( $node_s->safe_psql(
 	'subscriptions are created with the two-phase option enabled');
 
 # Confirm the pre-existing subscription has been removed
-$result = $node_s->safe_psql(
+$result = $node_s1->safe_psql(
 	'postgres', qq(
 	SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub'
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
 # Get subscription names
-$result = $node_s->safe_psql(
+$result = $node_s1->safe_psql(
 	'postgres', qq(
 	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
 ));
 my @subnames = split("\n", $result);
 
 # Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+$node_s1->wait_for_subscription_sync($node_p, $subnames[0]);
+$node_s1->wait_for_subscription_sync($node_p, $subnames[1]);
 
 # Confirm the failover slot has been removed
-$result = $node_s->safe_psql($db1,
+$result = $node_s1->safe_psql($db1,
 	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'"
 );
 is($result, qq(0), 'failover slot was removed');
 
 # Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+$result = $node_s1->safe_psql($db1, 'SELECT * FROM tbl1');
 is( $result, qq(first row
 second row
 third row),
 	"logical replication works in database $db1");
 
 # Check result in database $db2
-$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2');
+$result = $node_s1->safe_psql($db2, 'SELECT * FROM tbl2');
 is($result, qq(row 1), "logical replication works in database $db2");
 
 # Different system identifier?
 my $sysid_p = $node_p->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
-my $sysid_s = $node_s->safe_psql('postgres',
+my $sysid_s = $node_s1->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
 # clean up
 $node_p->teardown_node;
-$node_s->teardown_node;
+$node_s1->teardown_node;
+$node_s2->teardown_node;
 $node_t->teardown_node;
 $node_f->teardown_node;
 
-- 
2.34.1

#53Shubham Khanna
khannashubham1197@gmail.com
In reply to: Peter Smith (#49)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 21, 2025 at 5:18 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Thu, Feb 20, 2025 at 10:26 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch quickly!

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM

pg_subscription;");

+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so

that

very loop will see exactly the same content. Also, 'postgres' is also the user

database.

I feel you must ensure that all three databases (postgres, $db1, and $db2) have

a

subscription here.

Fixed.

My point was that the loop does not have meaning because pg_subscription
is a global one. I and Peter considered changes like [1] is needed. It ensures
that each databases have a subscription.
Note: [1] cannot pass the test as-is because $db1 and $db2 contains special
characters. Please escape appropriately.

Yes. Some test is still needed to confirm the expected subscriptions
all get created for respective dbs. But, the current test loop just
isn't doing it properly.

Fixed.

Other comments are listed in below.

01.
```
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
```

Peter's comment [2] does not say that option name should be changed.
The scope of his comment is only in the .c file.

Yes, that's correct. My v9 suggestion to change 'all' to 'all_dbs' was
referring only to the field name of struct CreateSubscriberOptions,
nothing else. Not the usage help, not the error messages, not the
docs, not the tests, not the commit message.

Fixed.

The attached patch at [1]/messages/by-id/CAHv8RjJ9BsCg+pur307b1JnfQebnmxFZLw4LdcGX7f-=6OK1vw@mail.gmail.com contains the suggested changes.

[1]: /messages/by-id/CAHv8RjJ9BsCg+pur307b1JnfQebnmxFZLw4LdcGX7f-=6OK1vw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#54Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#50)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 21, 2025 at 9:44 AM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Fri, Feb 21, 2025 at 5:18 AM Peter Smith <smithpb2250@gmail.com> wrote:

On Thu, Feb 20, 2025 at 10:26 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch quickly!

04.
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ($db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       my $sub_exists =
+         $node_s1->safe_psql($dbname, "SELECT count(*) FROM

pg_subscription;");

+       is($sub_exists, '3', "Subscription created successfully for $dbname");
+}
```

Hmm, what do you want to check here? pg_subscription is a global catalog so

that

very loop will see exactly the same content. Also, 'postgres' is also the user

database.

I feel you must ensure that all three databases (postgres, $db1, and $db2) have

a

subscription here.

Fixed.

My point was that the loop does not have meaning because pg_subscription
is a global one. I and Peter considered changes like [1] is needed. It ensures
that each databases have a subscription.
Note: [1] cannot pass the test as-is because $db1 and $db2 contains special
characters. Please escape appropriately.

Yes. Some test is still needed to confirm the expected subscriptions
all get created for respective dbs. But, the current test loop just
isn't doing it properly.

Other comments are listed in below.

01.
```
+     <term><option>-a</option></term>
+     <term><option>--all-dbs</option></term>
```

Peter's comment [2] does not say that option name should be changed.
The scope of his comment is only in the .c file.

Yes, that's correct. My v9 suggestion to change 'all' to 'all_dbs' was
referring only to the field name of struct CreateSubscriberOptions,
nothing else. Not the usage help, not the error messages, not the
docs, not the tests, not the commit message.

+1. We don't want yet another option to specify all databases. :)

Fixed.

The attached patch at [1]/messages/by-id/CAHv8RjJ9BsCg+pur307b1JnfQebnmxFZLw4LdcGX7f-=6OK1vw@mail.gmail.com contains the suggested changes.

[1]: /messages/by-id/CAHv8RjJ9BsCg+pur307b1JnfQebnmxFZLw4LdcGX7f-=6OK1vw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#55Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#51)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 21, 2025 at 10:18 AM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Wed, Feb 19, 2025 at 4:23 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch!

I have added a test case for non-dry-run mode to ensure that
replication slots, subscriptions, and publications work as expected
when '--all' is specified. Additionally, I have split the test file
into two parts:
1) Command-line sanity checks – Validates the correctness of option parsing.
2) '--all' functionality and behavior – Verifies the health of
subscriptions and ensures that --all specific scenarios, such as
non-template databases not being subscribed to, are properly handled.
This should improve test coverage while keeping the structure manageable.

TBH, I feel your change does not separate the test file into the two parts. ISTM
you just added some validation checks and a test how --all option works.
Ashutosh, does it match your suggestion?

Thanks Hayato for pointing it out. The test changes don't match my
expectations. As you rightly pointed out, I expected two (actually
three if needed) separate test files one for argument validation and
one for testing --database scenarios (the existing tests) and one more
for testing same scenarios when --all is specified. Right now all it
does is "# Verify that only user databases got subscriptions (not
template databases)". I expected testing the actual replication as
well like tests between lines around 527 and 582 in the latest
patchset. Those tests are performed when --database is subscribed. We
need similar tests performed when --all is specified. I didn't find
any of those tests being performed on node_s2. Given that the tests
for --databases and --all will be very similar, having them in the
same test file makes more sense. We also seem to be missing
$node_s2->teardown_node, do we?

I agree with your point. Right now, my focus is on fixing the patch
first, and I plan to split the test files at the end. That approach
might help streamline the process.
As of the latest patch attached at [1]/messages/by-id/CAHv8RjJ9BsCg+pur307b1JnfQebnmxFZLw4LdcGX7f-=6OK1vw@mail.gmail.com, this issue remains unresolved.
I will proceed with splitting the tests once all other issues are
addressed.

[1]: /messages/by-id/CAHv8RjJ9BsCg+pur307b1JnfQebnmxFZLw4LdcGX7f-=6OK1vw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#56Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#52)
1 attachment(s)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for updating the patch.

I think the modification [1]``` +# Verify that only user databases got subscriptions (not template databases) +my @user_dbs = ('postgres', $db1, $db2); +foreach my $dbname (@user_dbs) +{ + $result = $node_s2->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f';" + ); + is($result, '3', "Subscription created successfully for $dbname"); + $result = $node_s2->safe_psql('postgres', + "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';" + ); + is($result, '0', "Subscription created successfully for $dbname"); +} ``` is not correct - the loop is meaningless because the same
query would be executed every time. How about idea like attached? Here, instead of
try escaping dbname, dbname is directly obtained from the instance and they are compared.

How do you think?

[1]:
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ('postgres', $db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+	$result = $node_s2->safe_psql('postgres',
+		"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f';"
+	);
+	is($result, '3', "Subscription created successfully for $dbname");
+	$result = $node_s2->safe_psql('postgres',
+		"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
+	);
+	is($result, '0', "Subscription created successfully for $dbname");
+}
```

Best regards,
Hayato Kuroda
FUJITSU LIMITED

Attachments:

kuroda_fix.diffsapplication/octet-stream; name=kuroda_fix.diffsDownload
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 7574a3903e..79ef8b1764 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -496,19 +496,34 @@ command_ok(
 
 $node_s2->start;
 
-# Verify that only user databases got subscriptions (not template databases)
+# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both
+# $db1 and $db2 must be escaped to pass the sqfe_psql(), but it is difficult.
+# Thus, we define a cursor, obtain a dbname from the instance and compere one
+# by one.
 my @user_dbs = ('postgres', $db1, $db2);
+
+my $bgconn = $node_s2->background_psql('postgres');
+$bgconn->query_safe(
+	qq[
+	BEGIN;
+	DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f' ORDER BY pg_database.oid;
+]);
+
+# Fetch from the cursor three times and confirm the existence of the
+# subscription on $dbname
 foreach my $dbname (@user_dbs)
 {
-	$result = $node_s2->safe_psql('postgres',
-		"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f';"
-	);
-	is($result, '3', "Subscription created successfully for $dbname");
-	$result = $node_s2->safe_psql('postgres',
-		"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
-	);
-	is($result, '0', "Subscription created successfully for $dbname");
+	$result = $bgconn->query_safe("FETCH cursor;");
+
+	is($result, $dbname, "subscription is created on $dbname");
 }
+$bgconn->quit;
+
+# Verify template databases do not have subscriptions
+$result = $node_s2->safe_psql('postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
 
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
#57Ajin Cherian
itsajin@gmail.com
In reply to: Shubham Khanna (#52)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 28, 2025 at 11:34 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached Patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Some comments:
1.
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

Don't start the sentence with "Cannot". Change the sentence to "This
option cannot be used together with ..."
similar sentences used in 3 other places below this as well. Change all of them.

2.
+# Verify that only user databases got subscriptions (not template databases)

change to "Verify that only user databases have subscriptions"

regards,
Ajin Cherian
Fujitsu Australia

#58Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Ajin Cherian (#57)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 6, 2025 at 9:18 AM Ajin Cherian <itsajin@gmail.com> wrote:

+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

Don't start the sentence with "Cannot". Change the sentence to "This
option cannot be used together with ..."
similar sentences used in 3 other places below this as well. Change all of them.

I didn't find any instance of "Cannot be" in the *.sgml files, but I
find some instances of "Can be". So it seems we allow such constructs
in the documentation. Any reasons for suggesting this change?

--
Best Wishes,
Ashutosh Bapat

#59Ajin Cherian
itsajin@gmail.com
In reply to: Ashutosh Bapat (#58)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 10, 2025 at 3:58 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 6, 2025 at 9:18 AM Ajin Cherian <itsajin@gmail.com> wrote:

+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

Don't start the sentence with "Cannot". Change the sentence to "This
option cannot be used together with ..."
similar sentences used in 3 other places below this as well. Change all of them.

I didn't find any instance of "Cannot be" in the *.sgml files, but I
find some instances of "Can be". So it seems we allow such constructs
in the documentation. Any reasons for suggesting this change?

I just don't think it is correct English to start a sentence with
"Cannot be". I checked with grammarly as well.

regards,
Ajin Cherian
Fujitsu Australia

#60Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#56)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Feb 28, 2025 at 6:33 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

Thanks for updating the patch.

I think the modification [1] is not correct - the loop is meaningless because the same
query would be executed every time. How about idea like attached? Here, instead of
try escaping dbname, dbname is directly obtained from the instance and they are compared.

How do you think?

[1]:
```
+# Verify that only user databases got subscriptions (not template databases)
+my @user_dbs = ('postgres', $db1, $db2);
+foreach my $dbname (@user_dbs)
+{
+       $result = $node_s2->safe_psql('postgres',
+               "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f';"
+       );
+       is($result, '3', "Subscription created successfully for $dbname");
+       $result = $node_s2->safe_psql('postgres',
+               "SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
+       );
+       is($result, '0', "Subscription created successfully for $dbname");
+}
```

I agree with your suggestion and have incorporated the proposed
changes in the latest patch. Instead of escaping dbname, I now fetch
it directly from the instance for comparison, making the loop more
meaningful.

Additionally, as suggested by Ashutosh in [1]/messages/by-id/CAExHW5uJHYAge99oS_iPfGWwZ_eCr2xFCNnifQkGs2GXeMQKGQ@mail.gmail.com, I have split the
040_pg_createsubscriber.pl file into three parts to improve clarity.

The attached patch includes all the suggested changes.

[1]: /messages/by-id/CAExHW5uJHYAge99oS_iPfGWwZ_eCr2xFCNnifQkGs2GXeMQKGQ@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#61Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ajin Cherian (#57)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 6, 2025 at 9:18 AM Ajin Cherian <itsajin@gmail.com> wrote:

On Fri, Feb 28, 2025 at 11:34 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached Patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Some comments:
1.
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

Don't start the sentence with "Cannot". Change the sentence to "This
option cannot be used together with ..."
similar sentences used in 3 other places below this as well. Change all of them.

Fixed.

2.
+# Verify that only user databases got subscriptions (not template databases)

change to "Verify that only user databases have subscriptions"

Fixed.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v12-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v12-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From 7b3235812636c09dfc80f732017e46304e02ec4f Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 28 Feb 2025 17:29:30 +0530
Subject: [PATCH v12] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  53 ++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 ++++-
 .../t/040_pg_createsubscriber.pl              | 373 -----------------
 .../t/041_covert_standby_to_subscriber.pl     | 386 ++++++++++++++++++
 src/bin/pg_basebackup/t/042_all_option.pl     | 217 ++++++++++
 5 files changed, 738 insertions(+), 383 deletions(-)
 create mode 100644 src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl
 create mode 100644 src/bin/pg_basebackup/t/042_all_option.pl

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index b4b996236e4..a1abd64fd75 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,9 +131,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.
       </para>
      </listitem>
@@ -230,7 +268,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -246,7 +285,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -261,7 +301,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index a5a2d61165d..d2771a81780 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -44,6 +44,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 /* per-database publication/subscription info */
@@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -234,6 +236,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1902,11 +1905,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1976,6 +2028,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1993,11 +2046,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2085,6 +2141,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options. */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2138,14 +2216,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c35fa108ce3..d6e97405ee1 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -15,32 +15,6 @@ program_options_handling_ok('pg_createsubscriber');
 
 my $datadir = PostgreSQL::Test::Utils::tempdir;
 
-# Generate a database with a name made of a range of ASCII characters.
-# Extracted from 002_pg_upgrade.pl.
-sub generate_db
-{
-	my ($node, $prefix, $from_char, $to_char, $suffix) = @_;
-
-	my $dbname = $prefix;
-	for my $i ($from_char .. $to_char)
-	{
-		next if $i == 7 || $i == 10 || $i == 13;    # skip BEL, LF, and CR
-		$dbname = $dbname . sprintf('%c', $i);
-	}
-
-	$dbname .= $suffix;
-
-	# On Windows, older IPC::Run versions can mis-quote command line arguments
-	# containing double quote or backslash
-	$dbname =~ tr/\"\\//d if ($windows_os);
-
-	$node->command_ok(
-		[ 'createdb', $dbname ],
-		"created database with ASCII characters from $from_char to $to_char");
-
-	return $dbname;
-}
-
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -118,351 +92,4 @@ command_fails(
 	],
 	'wrong number of replication slot names');
 
-# Set up node P as primary
-my $node_p = PostgreSQL::Test::Cluster->new('node_p');
-my $pconnstr = $node_p->connstr;
-$node_p->init(allows_streaming => 'logical');
-# Disable autovacuum to avoid generating xid during stats update as otherwise
-# the new XID could then be replicated to standby at some random point making
-# slots at primary lag behind standby during slot sync.
-$node_p->append_conf('postgresql.conf', 'autovacuum = off');
-$node_p->start;
-
-# Set up node F as about-to-fail node
-# Force it to initialize a new cluster instead of copying a
-# previously initdb'd cluster. New cluster has a different system identifier so
-# we can test if the target cluster is a copy of the source cluster.
-my $node_f = PostgreSQL::Test::Cluster->new('node_f');
-$node_f->init(force_initdb => 1, allows_streaming => 'logical');
-
-# On node P
-# - create databases
-# - create test tables
-# - insert a row
-# - create a physical replication slot
-my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
-my $db2 = generate_db($node_p, 'regression', 46, 90, '');
-
-$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)');
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')");
-$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)');
-my $slotname = 'physical_slot';
-$node_p->safe_psql($db2,
-	"SELECT pg_create_physical_replication_slot('$slotname')");
-
-# Set up node S as standby linking to node P
-$node_p->backup('backup_1');
-my $node_s = PostgreSQL::Test::Cluster->new('node_s');
-$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
-$node_s->append_conf(
-	'postgresql.conf', qq[
-primary_slot_name = '$slotname'
-primary_conninfo = '$pconnstr dbname=postgres'
-hot_standby_feedback = on
-]);
-$node_s->set_standby_mode();
-$node_s->start;
-
-# Set up node T as standby linking to node P then promote it
-my $node_t = PostgreSQL::Test::Cluster->new('node_t');
-$node_t->init_from_backup($node_p, 'backup_1', has_streaming => 1);
-$node_t->set_standby_mode();
-$node_t->start;
-$node_t->promote;
-$node_t->stop;
-
-# Run pg_createsubscriber on a promoted server
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_t->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_t->host,
-		'--subscriber-port' => $node_t->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'target server is not in recovery');
-
-# Run pg_createsubscriber when standby is running
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'standby is up and running');
-
-# Run pg_createsubscriber on about-to-fail node F
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--pgdata' => $node_f->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_f->host,
-		'--subscriber-port' => $node_f->port,
-		'--database' => $db1,
-		'--database' => $db2
-	],
-	'subscriber data directory is not a copy of the source database cluster');
-
-# Set up node C as standby linking to node S
-$node_s->backup('backup_2');
-my $node_c = PostgreSQL::Test::Cluster->new('node_c');
-$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1);
-$node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef);
-$node_c->set_standby_mode();
-
-# Run pg_createsubscriber on node C (P -> S -> C)
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_c->data_dir,
-		'--publisher-server' => $node_s->connstr($db1),
-		'--socketdir' => $node_c->host,
-		'--subscriber-port' => $node_c->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'primary server is in recovery');
-
-# Check some unmet conditions on node P
-$node_p->append_conf(
-	'postgresql.conf', q{
-wal_level = replica
-max_replication_slots = 1
-max_wal_senders = 1
-max_worker_processes = 2
-});
-$node_p->restart;
-$node_s->stop;
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
-		'--database' => $db2,
-
-	],
-	'primary contains unmet conditions on node P');
-# Restore default settings here but only apply it after testing standby. Some
-# standby settings should not be a lower setting than on the primary.
-$node_p->append_conf(
-	'postgresql.conf', q{
-wal_level = logical
-max_replication_slots = 10
-max_wal_senders = 10
-max_worker_processes = 8
-});
-
-# Check some unmet conditions on node S
-$node_s->append_conf(
-	'postgresql.conf', q{
-max_replication_slots = 1
-max_logical_replication_workers = 1
-max_worker_processes = 2
-});
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'standby contains unmet conditions on node S');
-$node_s->append_conf(
-	'postgresql.conf', q{
-max_replication_slots = 10
-max_logical_replication_workers = 4
-max_worker_processes = 8
-});
-# Restore default settings on both servers
-$node_p->restart;
-
-# Create failover slot to test its removal
-my $fslotname = 'failover_slot';
-$node_p->safe_psql($db1,
-	"SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)"
-);
-$node_s->start;
-# Wait for the standby to catch up so that the standby is not lagging behind
-# the failover slot.
-$node_p->wait_for_replay_catchup($node_s);
-$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()");
-my $result = $node_s->safe_psql('postgres',
-	"SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary"
-);
-is($result, 'failover_slot', 'failover slot is synced');
-
-# Insert another row on node P and wait node S to catch up. We
-# intentionally performed this insert after syncing logical slot
-# as otherwise the local slot's (created during synchronization of
-# slot) xmin on standby could be ahead of the remote slot leading
-# to failure in synchronization.
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')");
-$node_p->wait_for_replay_catchup($node_s);
-
-# Create subscription to test its removal
-my $dummy_sub = 'regress_sub_dummy';
-$node_p->safe_psql($db1,
-	"CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)"
-);
-$node_p->wait_for_replay_catchup($node_s);
-$node_s->stop;
-
-# dry run mode on node S
-command_ok(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--publication' => 'pub1',
-		'--publication' => 'pub2',
-		'--subscription' => 'sub1',
-		'--subscription' => 'sub2',
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'run pg_createsubscriber --dry-run on node S');
-
-# Check if node S is still a standby
-$node_s->start;
-is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'),
-	't', 'standby is in recovery');
-$node_s->stop;
-
-# pg_createsubscriber can run without --databases option
-command_ok(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--replication-slot' => 'replslot1',
-	],
-	'run pg_createsubscriber without --databases');
-
-# Run pg_createsubscriber on node S.  --verbose is used twice
-# to show more information.
-# In passing, also test the --enable-two-phase option
-command_ok(
-	[
-		'pg_createsubscriber',
-		'--verbose', '--verbose',
-		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--publication' => 'pub1',
-		'--publication' => 'pub2',
-		'--replication-slot' => 'replslot1',
-		'--replication-slot' => 'replslot2',
-		'--database' => $db1,
-		'--database' => $db2,
-		'--enable-two-phase'
-	],
-	'run pg_createsubscriber on node S');
-
-# Confirm the physical replication slot has been removed
-$result = $node_p->safe_psql($db1,
-	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$slotname'"
-);
-is($result, qq(0),
-	'the physical replication slot used as primary_slot_name has been removed'
-);
-
-# Insert rows on P
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')");
-$node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')");
-
-# Start subscriber
-$node_s->start;
-
-# Verify that all subtwophase states are pending or enabled,
-# e.g. there are no subscriptions where subtwophase is disabled ('d')
-is( $node_s->safe_psql(
-		'postgres',
-		"SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'"
-	),
-	't',
-	'subscriptions are created with the two-phase option enabled');
-
-# Confirm the pre-existing subscription has been removed
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub'
-));
-is($result, qq(0), 'pre-existing subscription was dropped');
-
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
-
-# Confirm the failover slot has been removed
-$result = $node_s->safe_psql($db1,
-	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'"
-);
-is($result, qq(0), 'failover slot was removed');
-
-# Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
-is( $result, qq(first row
-second row
-third row),
-	"logical replication works in database $db1");
-
-# Check result in database $db2
-$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2');
-is($result, qq(row 1), "logical replication works in database $db2");
-
-# Different system identifier?
-my $sysid_p = $node_p->safe_psql('postgres',
-	'SELECT system_identifier FROM pg_control_system()');
-my $sysid_s = $node_s->safe_psql('postgres',
-	'SELECT system_identifier FROM pg_control_system()');
-ok($sysid_p != $sysid_s, 'system identifier was changed');
-
-# clean up
-$node_p->teardown_node;
-$node_s->teardown_node;
-$node_t->teardown_node;
-$node_f->teardown_node;
-
 done_testing();
diff --git a/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl b/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl
new file mode 100644
index 00000000000..a51c86ed9f3
--- /dev/null
+++ b/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl
@@ -0,0 +1,386 @@
+# Copyright (c) 2024-2025, PostgreSQL Global Development Group
+
+#
+# Test using a standby server as the subscriber.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Generate a database with a name made of a range of ASCII characters.
+# Extracted from 002_pg_upgrade.pl.
+sub generate_db
+{
+	my ($node, $prefix, $from_char, $to_char, $suffix) = @_;
+
+	my $dbname = $prefix;
+	for my $i ($from_char .. $to_char)
+	{
+		next if $i == 7 || $i == 10 || $i == 13;    # skip BEL, LF, and CR
+		$dbname = $dbname . sprintf('%c', $i);
+	}
+
+	$dbname .= $suffix;
+
+	# On Windows, older IPC::Run versions can mis-quote command line arguments
+	# containing double quote or backslash
+	$dbname =~ tr/\"\\//d if ($windows_os);
+
+	$node->command_ok(
+		[ 'createdb', $dbname ],
+		"created database with ASCII characters from $from_char to $to_char");
+
+	return $dbname;
+}
+
+#
+# Set up node P as primary
+my $node_p = PostgreSQL::Test::Cluster->new('node_p');
+my $pconnstr = $node_p->connstr;
+$node_p->init(allows_streaming => 'logical');
+# Disable autovacuum to avoid generating xid during stats update as otherwise
+# the new XID could then be replicated to standby at some random point making
+# slots at primary lag behind standby during slot sync.
+$node_p->append_conf('postgresql.conf', 'autovacuum = off');
+$node_p->start;
+
+# Set up node F as about-to-fail node
+# Force it to initialize a new cluster instead of copying a
+# previously initdb'd cluster. New cluster has a different system identifier so
+# we can test if the target cluster is a copy of the source cluster.
+my $node_f = PostgreSQL::Test::Cluster->new('node_f');
+$node_f->init(force_initdb => 1, allows_streaming => 'logical');
+
+# On node P
+# - create databases
+# - create test tables
+# - insert a row
+# - create a physical replication slot
+my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
+my $db2 = generate_db($node_p, 'regression', 46, 90, '');
+
+$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)');
+my $slotname = 'physical_slot';
+$node_p->safe_psql($db2,
+	"SELECT pg_create_physical_replication_slot('$slotname')");
+
+# Set up node S as standby linking to node P
+$node_p->backup('backup_1');
+my $node_s = PostgreSQL::Test::Cluster->new('node_s');
+$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_s->append_conf(
+	'postgresql.conf', qq[
+primary_slot_name = '$slotname'
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s->set_standby_mode();
+$node_s->start;
+
+# Set up node T as standby linking to node P then promote it
+my $node_t = PostgreSQL::Test::Cluster->new('node_t');
+$node_t->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_t->set_standby_mode();
+$node_t->start;
+$node_t->promote;
+$node_t->stop;
+
+# Run pg_createsubscriber on a promoted server
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_t->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_t->host,
+		'--subscriber-port' => $node_t->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'target server is not in recovery');
+
+# Run pg_createsubscriber when standby is running
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'standby is up and running');
+
+# Run pg_createsubscriber on about-to-fail node F
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_f->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_f->host,
+		'--subscriber-port' => $node_f->port,
+		'--database' => $db1,
+		'--database' => $db2
+	],
+	'subscriber data directory is not a copy of the source database cluster');
+
+# Set up node C as standby linking to node S
+$node_s->backup('backup_2');
+my $node_c = PostgreSQL::Test::Cluster->new('node_c');
+$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1);
+$node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef);
+$node_c->set_standby_mode();
+
+# Run pg_createsubscriber on node C (P -> S -> C)
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_c->data_dir,
+		'--publisher-server' => $node_s->connstr($db1),
+		'--socketdir' => $node_c->host,
+		'--subscriber-port' => $node_c->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'primary server is in recovery');
+
+# Check some unmet conditions on node P
+$node_p->append_conf(
+	'postgresql.conf', q{
+wal_level = replica
+max_replication_slots = 1
+max_wal_senders = 1
+max_worker_processes = 2
+});
+$node_p->restart;
+$node_s->stop;
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--database' => $db2,
+
+	],
+	'primary contains unmet conditions on node P');
+# Restore default settings here but only apply it after testing standby. Some
+# standby settings should not be a lower setting than on the primary.
+$node_p->append_conf(
+	'postgresql.conf', q{
+wal_level = logical
+max_replication_slots = 10
+max_wal_senders = 10
+max_worker_processes = 8
+});
+
+# Check some unmet conditions on node S
+$node_s->append_conf(
+	'postgresql.conf', q{
+max_replication_slots = 1
+max_logical_replication_workers = 1
+max_worker_processes = 2
+});
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'standby contains unmet conditions on node S');
+$node_s->append_conf(
+	'postgresql.conf', q{
+max_replication_slots = 10
+max_logical_replication_workers = 4
+max_worker_processes = 8
+});
+# Restore default settings on both servers
+$node_p->restart;
+
+# Create failover slot to test its removal
+my $fslotname = 'failover_slot';
+$node_p->safe_psql($db1,
+	"SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)"
+);
+$node_s->start;
+# Wait for the standby to catch up so that the standby is not lagging behind
+# the failover slot.
+$node_p->wait_for_replay_catchup($node_s);
+$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()");
+my $result = $node_s->safe_psql('postgres',
+	"SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary"
+);
+is($result, 'failover_slot', 'failover slot is synced');
+
+# Insert another row on node P and wait node S to catch up. We
+# intentionally performed this insert after syncing logical slot
+# as otherwise the local slot's (created during synchronization of
+# slot) xmin on standby could be ahead of the remote slot leading
+# to failure in synchronization.
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')");
+$node_p->wait_for_replay_catchup($node_s);
+
+# Create subscription to test its removal
+my $dummy_sub = 'regress_sub_dummy';
+$node_p->safe_psql($db1,
+	"CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)"
+);
+$node_p->wait_for_replay_catchup($node_s);
+$node_s->stop;
+
+# dry run mode on node S
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--publication' => 'pub2',
+		'--subscription' => 'sub1',
+		'--subscription' => 'sub2',
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'run pg_createsubscriber --dry-run on node S');
+
+# Check if node S is still a standby
+$node_s->start;
+is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'),
+	't', 'standby is in recovery');
+$node_s->stop;
+
+# pg_createsubscriber can run without --databases option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+	],
+	'run pg_createsubscriber without --databases');
+
+# Run pg_createsubscriber on node S.  --verbose is used twice
+# to show more information.
+# In passing, also test the --enable-two-phase option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose', '--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--publication' => 'pub2',
+		'--replication-slot' => 'replslot1',
+		'--replication-slot' => 'replslot2',
+		'--database' => $db1,
+		'--database' => $db2,
+		'--enable-two-phase'
+	],
+	'run pg_createsubscriber on node S');
+
+# Confirm the physical replication slot has been removed
+$result = $node_p->safe_psql($db1,
+	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$slotname'"
+);
+is($result, qq(0),
+	'the physical replication slot used as primary_slot_name has been removed'
+);
+
+# Insert rows on P
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')");
+
+# Start subscriber
+$node_s->start;
+
+# Verify that all subtwophase states are pending or enabled,
+# e.g. there are no subscriptions where subtwophase is disabled ('d')
+is( $node_s->safe_psql(
+		'postgres',
+		"SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'"
+	),
+	't',
+	'subscriptions are created with the two-phase option enabled');
+
+# Confirm the pre-existing subscription has been removed
+$result = $node_s->safe_psql(
+	'postgres', qq(
+	SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub'
+));
+is($result, qq(0), 'pre-existing subscription was dropped');
+
+# Get subscription names
+$result = $node_s->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
+$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+
+# Confirm the failover slot has been removed
+$result = $node_s->safe_psql($db1,
+	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'"
+);
+is($result, qq(0), 'failover slot was removed');
+
+# Check result in database $db1
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2
+$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2');
+is($result, qq(row 1), "logical replication works in database $db2");
+
+# Different system identifier?
+my $sysid_p = $node_p->safe_psql('postgres',
+	'SELECT system_identifier FROM pg_control_system()');
+my $sysid_s = $node_s->safe_psql('postgres',
+	'SELECT system_identifier FROM pg_control_system()');
+ok($sysid_p != $sysid_s, 'system identifier was changed');
+
+# clean up
+$node_p->teardown_node;
+$node_s->teardown_node;
+$node_t->teardown_node;
+$node_f->teardown_node;
+
+done_testing();
diff --git a/src/bin/pg_basebackup/t/042_all_option.pl b/src/bin/pg_basebackup/t/042_all_option.pl
new file mode 100644
index 00000000000..b228ed612b7
--- /dev/null
+++ b/src/bin/pg_basebackup/t/042_all_option.pl
@@ -0,0 +1,217 @@
+# Copyright (c) 2024-2025, PostgreSQL Global Development Group
+
+#
+# Test using a standby server as the subscriber.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Generate a database with a name made of a range of ASCII characters.
+# Extracted from 002_pg_upgrade.pl.
+sub generate_db
+{
+	my ($node, $prefix, $from_char, $to_char, $suffix) = @_;
+
+	my $dbname = $prefix;
+	for my $i ($from_char .. $to_char)
+	{
+		next if $i == 7 || $i == 10 || $i == 13;    # skip BEL, LF, and CR
+		$dbname = $dbname . sprintf('%c', $i);
+	}
+
+	$dbname .= $suffix;
+
+	# On Windows, older IPC::Run versions can mis-quote command line arguments
+	# containing double quote or backslash
+	$dbname =~ tr/\"\\//d if ($windows_os);
+
+	$node->command_ok(
+		[ 'createdb', $dbname ],
+		"created database with ASCII characters from $from_char to $to_char");
+
+	return $dbname;
+}
+
+#
+# Set up node P as primary
+my $node_p = PostgreSQL::Test::Cluster->new('node_p');
+my $pconnstr = $node_p->connstr;
+$node_p->init(allows_streaming => 'logical');
+# Disable autovacuum to avoid generating xid during stats update as otherwise
+# the new XID could then be replicated to standby at some random point making
+# slots at primary lag behind standby during slot sync.
+$node_p->append_conf('postgresql.conf', 'autovacuum = off');
+$node_p->start;
+
+# On node P
+# - create databases
+# - create test tables
+# - insert a row
+my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
+my $db2 = generate_db($node_p, 'regression', 46, 90, '');
+
+$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)');
+
+# Set up node S as standby linking to node P
+$node_p->backup('backup_1');
+my $node_s = PostgreSQL::Test::Cluster->new('node_s');
+$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_s->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s->set_standby_mode();
+
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
+# run pg_createsubscriber with '--all' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_s->start;
+
+# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both
+# $db1 and $db2 must be escaped to pass the safe_psql(), but it is difficult.
+# Thus, we define a cursor, obtain a dbname from the instance and compere one
+# by one.
+my @user_dbs = ('postgres', $db1, $db2);
+
+my $bgconn = $node_s->background_psql('postgres');
+$bgconn->query_safe(
+	qq[
+	BEGIN;
+	DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f' ORDER BY pg_database.oid;
+]);
+
+# Fetch from the cursor three times and confirm the existence of the
+# subscription on $dbname
+foreach my $dbname (@user_dbs)
+{
+	my $result = $bgconn->query_safe("FETCH cursor;");
+
+	is($result, $dbname, "subscription is created on $dbname");
+}
+$bgconn->quit;
+
+# Verify template databases do not have subscriptions
+my $result = $node_s->safe_psql('postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# clean up
+$node_p->teardown_node;
+$node_s->teardown_node;
+
+done_testing();
-- 
2.41.0.windows.3

#62Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#58)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 10, 2025 at 10:28 AM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 6, 2025 at 9:18 AM Ajin Cherian <itsajin@gmail.com> wrote:

+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

Don't start the sentence with "Cannot". Change the sentence to "This
option cannot be used together with ..."
similar sentences used in 3 other places below this as well. Change all of them.

I didn't find any instance of "Cannot be" in the *.sgml files, but I
find some instances of "Can be". So it seems we allow such constructs
in the documentation. Any reasons for suggesting this change?

--

Updated the .sgml file accordingly. The attached patch at [1]/messages/by-id/CAHv8Rj+37ja589BzqB5bz0ZYGWb5gpnP9of8SoqKc=DqLmvxBg@mail.gmail.com contains
the required changes.

[1]: /messages/by-id/CAHv8Rj+37ja589BzqB5bz0ZYGWb5gpnP9of8SoqKc=DqLmvxBg@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#63Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ajin Cherian (#59)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 10, 2025 at 11:04 AM Ajin Cherian <itsajin@gmail.com> wrote:

On Mon, Mar 10, 2025 at 3:58 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 6, 2025 at 9:18 AM Ajin Cherian <itsajin@gmail.com> wrote:

+       Subscription names, publication names, and replication slot names are
+       automatically generated. Cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

Don't start the sentence with "Cannot". Change the sentence to "This
option cannot be used together with ..."
similar sentences used in 3 other places below this as well. Change all of them.

I didn't find any instance of "Cannot be" in the *.sgml files, but I
find some instances of "Can be". So it seems we allow such constructs
in the documentation. Any reasons for suggesting this change?

I just don't think it is correct English to start a sentence with
"Cannot be". I checked with grammarly as well.

Fixed.

The attached patch at [1]/messages/by-id/CAHv8Rj+37ja589BzqB5bz0ZYGWb5gpnP9of8SoqKc=DqLmvxBg@mail.gmail.com contains the required changes.

[1]: /messages/by-id/CAHv8Rj+37ja589BzqB5bz0ZYGWb5gpnP9of8SoqKc=DqLmvxBg@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#64Nisha Moond
nisha.moond412@gmail.com
In reply to: Shubham Khanna (#63)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi Shubham,

Here are a few comments for the v12 patch.

doc/src/sgml/ref/pg_createsubscriber.sgml :

1.
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.

is “create subscriptions for” repeated? The sentence doesn’t make sense.

2. Typo
+ switches. This option cannot be together with <option>--all</option>.

/cannot be together/cannot be used together/
~~~

3. src/bin/pg_basebackup/pg_createsubscriber.c :
+ /* Establish a connection to the PostgreSQL server */
+ conn = connect_database(opt->pub_conninfo_str, true);

I think comment will be more clear if say “ Establish a connection to
the primary server */ or “source server”
~~~

src/bin/pg_basebackup/t/042_all_option.pl :

4. As per Ashutosh's comment at [1]/messages/by-id/CAExHW5uJHYAge99oS_iPfGWwZ_eCr2xFCNnifQkGs2GXeMQKGQ@mail.gmail.com, tests need to be added to verify
logical replication behavior after using the --all option.
~~~~

Please refer to the attached top-up fix patch, which includes the
above changes along with some cosmetic fixes in the test file
042_all_option.pl.

[1]: /messages/by-id/CAExHW5uJHYAge99oS_iPfGWwZ_eCr2xFCNnifQkGs2GXeMQKGQ@mail.gmail.com

--
Thanks,
Nisha

Attachments:

0001-Nisha_v12_fixes.patchapplication/octet-stream; name=0001-Nisha_v12_fixes.patchDownload
From bbaad161d5add2f895e99cf087accc2558fde551 Mon Sep 17 00:00:00 2001
From: Nisha Moond <nisha.moond412@gmail.com>
Date: Fri, 14 Mar 2025 16:52:29 +0530
Subject: [PATCH] Nisha_v12_fixes

---
 doc/src/sgml/ref/pg_createsubscriber.sgml   |  5 +-
 src/bin/pg_basebackup/pg_createsubscriber.c |  4 +-
 src/bin/pg_basebackup/t/042_all_option.pl   | 65 ++++++++++++++++-----
 3 files changed, 55 insertions(+), 19 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index a1abd64fd75..7376c61f3eb 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -114,8 +114,7 @@ PostgreSQL documentation
      <listitem>
       <para>
        For all source server non-template databases create subscriptions for
-       create subscriptions for databases with the same names on the
-       target server.
+       databases with the same names on the target server.
        Subscription names, publication names, and replication slot names are
        automatically generated. This option cannot be used together with
        <option>--database</option>, <option>--publication</option>,
@@ -131,7 +130,7 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. This option cannot be together with <option>--all</option>.
+       switches. This option cannot be used together with <option>--all</option>.
        If <option>-d</option> option is not provided, the database name will be
        obtained from <option>-P</option> option. If the database name is not
        specified in either the <option>-d</option> option or
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index d2771a81780..efaef97bd3a 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -1916,7 +1916,7 @@ fetch_source_databases(struct CreateSubscriberOptions *opt)
 	PGconn	   *conn;
 	PGresult   *res;
 
-	/* Establish a connection to the PostgreSQL server */
+	/* Establish a connection to the source server */
 	conn = connect_database(opt->pub_conninfo_str, true);
 
 	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
@@ -2141,7 +2141,7 @@ main(int argc, char **argv)
 		}
 	}
 
-	/* Validate that --all is not used with incompatible options. */
+	/* Validate that --all is not used with incompatible options */
 	if (opt.all_dbs)
 	{
 		char	   *bad_switch = NULL;
diff --git a/src/bin/pg_basebackup/t/042_all_option.pl b/src/bin/pg_basebackup/t/042_all_option.pl
index b228ed612b7..18e74c294b9 100644
--- a/src/bin/pg_basebackup/t/042_all_option.pl
+++ b/src/bin/pg_basebackup/t/042_all_option.pl
@@ -53,9 +53,10 @@ $node_p->start;
 my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
 my $db2 = generate_db($node_p, 'regression', 46, 90, '');
 
-$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)');
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')");
-$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');
 
 # Set up node S as standby linking to node P
 $node_p->backup('backup_1');
@@ -85,7 +86,7 @@ command_fails_like(
 	qr/--database cannot be used with --all/,
 	'fail if --database is used with --all');
 
-# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# run pg_createsubscriber with '--all' and '--database' without '--dry-run'
 # and verify the failure
 command_fails_like(
 	[
@@ -95,13 +96,13 @@ command_fails_like(
 		'--publisher-server' => $node_p->connstr($db1),
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
 		'--all',
+		'--database' => $db1,
 	],
 	qr/--database cannot be used with --all/,
-	'fail if --database is used with --all');
+	'fail if --database is used with --all without --dry-run');
 
-# run pg_createsubscriber with '--publication' and '--all' and verify
+# run pg_createsubscriber with '--all' and '--publication' and verify
 # the failure
 command_fails_like(
 	[
@@ -118,7 +119,7 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
-# run pg_createsubscriber with '--replication-slot' and '--all' and
+# run pg_createsubscriber with '--all' and '--replication-slot' and
 # verify the failure
 command_fails_like(
 	[
@@ -129,13 +130,13 @@ command_fails_like(
 		'--publisher-server' => $node_p->connstr($db1),
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
-		'--replication-slot' => 'replslot1',
 		'--all',
+		'--replication-slot' => 'replslot1',
 	],
 	qr/--replication-slot cannot be used with --all/,
 	'fail if --replication-slot is used with --all');
 
-# run pg_createsubscriber with '--subscription' and '--all' and
+# run pg_createsubscriber with '--all' and '--subscription' and
 # verify the failure
 command_fails_like(
 	[
@@ -177,7 +178,7 @@ command_ok(
 		'--subscriber-port' => $node_s->port,
 		'--all',
 	],
-	'run pg_createsubscriber with --all');
+	'run pg_createsubscriber with --all without --dry-run');
 
 $node_s->start;
 
@@ -191,7 +192,9 @@ my $bgconn = $node_s->background_psql('postgres');
 $bgconn->query_safe(
 	qq[
 	BEGIN;
-	DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 'f' ORDER BY pg_database.oid;
+	DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database
+	WHERE subdbid = pg_database.oid and datistemplate = 'f'
+	ORDER BY pg_database.oid;
 ]);
 
 # Fetch from the cursor three times and confirm the existence of the
@@ -205,11 +208,45 @@ foreach my $dbname (@user_dbs)
 $bgconn->quit;
 
 # Verify template databases do not have subscriptions
-my $result = $node_s->safe_psql('postgres',
-	"SELECT count(*) FROM pg_subscription, pg_database WHERE subdbid = pg_database.oid and datistemplate = 't';"
+my $result = $node_s->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
 );
 is($result, '0', 'subscription is not created on template databases');
 
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')");
+
+# Get subscription names
+$result = $node_s->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
+$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+$node_s->wait_for_subscription_sync($node_p, $subnames[2]);
+
+# Check result in database 'postgres' of node S
+$result = $node_s->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node S
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2 of node S
+$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
-- 
2.34.1

#65Shubham Khanna
khannashubham1197@gmail.com
In reply to: Nisha Moond (#64)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Mar 14, 2025 at 5:43 PM Nisha Moond <nisha.moond412@gmail.com> wrote:

Hi Shubham,

Here are a few comments for the v12 patch.

doc/src/sgml/ref/pg_createsubscriber.sgml :

1.
+      <para>
+       For all source server non-template databases create subscriptions for
+       create subscriptions for databases with the same names on the
+       target server.

is “create subscriptions for” repeated? The sentence doesn’t make sense.

Fixed.

2. Typo
+ switches. This option cannot be together with <option>--all</option>.

/cannot be together/cannot be used together/
~~~

Fixed.

3. src/bin/pg_basebackup/pg_createsubscriber.c :
+ /* Establish a connection to the PostgreSQL server */
+ conn = connect_database(opt->pub_conninfo_str, true);

I think comment will be more clear if say “ Establish a connection to
the primary server */ or “source server”
~~~

Fixed.

src/bin/pg_basebackup/t/042_all_option.pl :

4. As per Ashutosh's comment at [1], tests need to be added to verify
logical replication behavior after using the --all option.
~~~~

Please refer to the attached top-up fix patch, which includes the
above changes along with some cosmetic fixes in the test file
042_all_option.pl.

[1] /messages/by-id/CAExHW5uJHYAge99oS_iPfGWwZ_eCr2xFCNnifQkGs2GXeMQKGQ@mail.gmail.com

--

Fixed.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v13-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v13-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From 82fa4302fd7e2319bbdb89049a702b5eb02d12ee Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 28 Feb 2025 17:29:30 +0530
Subject: [PATCH v13] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 ++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 ++++-
 .../t/040_pg_createsubscriber.pl              | 373 -----------------
 .../t/041_covert_standby_to_subscriber.pl     | 386 ++++++++++++++++++
 src/bin/pg_basebackup/t/042_all_option.pl     | 254 ++++++++++++
 5 files changed, 774 insertions(+), 383 deletions(-)
 create mode 100644 src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl
 create mode 100644 src/bin/pg_basebackup/t/042_all_option.pl

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index b4b996236e4..7376c61f3eb 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,21 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,9 +130,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.
       </para>
      </listitem>
@@ -230,7 +267,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -246,7 +284,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -261,7 +300,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 6baf92e8024..f918a0e33b8 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -44,6 +44,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 /* per-database publication/subscription info */
@@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -236,6 +238,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1904,11 +1907,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1978,6 +2030,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1995,11 +2048,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2087,6 +2143,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2140,14 +2218,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c35fa108ce3..d6e97405ee1 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -15,32 +15,6 @@ program_options_handling_ok('pg_createsubscriber');
 
 my $datadir = PostgreSQL::Test::Utils::tempdir;
 
-# Generate a database with a name made of a range of ASCII characters.
-# Extracted from 002_pg_upgrade.pl.
-sub generate_db
-{
-	my ($node, $prefix, $from_char, $to_char, $suffix) = @_;
-
-	my $dbname = $prefix;
-	for my $i ($from_char .. $to_char)
-	{
-		next if $i == 7 || $i == 10 || $i == 13;    # skip BEL, LF, and CR
-		$dbname = $dbname . sprintf('%c', $i);
-	}
-
-	$dbname .= $suffix;
-
-	# On Windows, older IPC::Run versions can mis-quote command line arguments
-	# containing double quote or backslash
-	$dbname =~ tr/\"\\//d if ($windows_os);
-
-	$node->command_ok(
-		[ 'createdb', $dbname ],
-		"created database with ASCII characters from $from_char to $to_char");
-
-	return $dbname;
-}
-
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -118,351 +92,4 @@ command_fails(
 	],
 	'wrong number of replication slot names');
 
-# Set up node P as primary
-my $node_p = PostgreSQL::Test::Cluster->new('node_p');
-my $pconnstr = $node_p->connstr;
-$node_p->init(allows_streaming => 'logical');
-# Disable autovacuum to avoid generating xid during stats update as otherwise
-# the new XID could then be replicated to standby at some random point making
-# slots at primary lag behind standby during slot sync.
-$node_p->append_conf('postgresql.conf', 'autovacuum = off');
-$node_p->start;
-
-# Set up node F as about-to-fail node
-# Force it to initialize a new cluster instead of copying a
-# previously initdb'd cluster. New cluster has a different system identifier so
-# we can test if the target cluster is a copy of the source cluster.
-my $node_f = PostgreSQL::Test::Cluster->new('node_f');
-$node_f->init(force_initdb => 1, allows_streaming => 'logical');
-
-# On node P
-# - create databases
-# - create test tables
-# - insert a row
-# - create a physical replication slot
-my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
-my $db2 = generate_db($node_p, 'regression', 46, 90, '');
-
-$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)');
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')");
-$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)');
-my $slotname = 'physical_slot';
-$node_p->safe_psql($db2,
-	"SELECT pg_create_physical_replication_slot('$slotname')");
-
-# Set up node S as standby linking to node P
-$node_p->backup('backup_1');
-my $node_s = PostgreSQL::Test::Cluster->new('node_s');
-$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
-$node_s->append_conf(
-	'postgresql.conf', qq[
-primary_slot_name = '$slotname'
-primary_conninfo = '$pconnstr dbname=postgres'
-hot_standby_feedback = on
-]);
-$node_s->set_standby_mode();
-$node_s->start;
-
-# Set up node T as standby linking to node P then promote it
-my $node_t = PostgreSQL::Test::Cluster->new('node_t');
-$node_t->init_from_backup($node_p, 'backup_1', has_streaming => 1);
-$node_t->set_standby_mode();
-$node_t->start;
-$node_t->promote;
-$node_t->stop;
-
-# Run pg_createsubscriber on a promoted server
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_t->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_t->host,
-		'--subscriber-port' => $node_t->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'target server is not in recovery');
-
-# Run pg_createsubscriber when standby is running
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'standby is up and running');
-
-# Run pg_createsubscriber on about-to-fail node F
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--pgdata' => $node_f->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_f->host,
-		'--subscriber-port' => $node_f->port,
-		'--database' => $db1,
-		'--database' => $db2
-	],
-	'subscriber data directory is not a copy of the source database cluster');
-
-# Set up node C as standby linking to node S
-$node_s->backup('backup_2');
-my $node_c = PostgreSQL::Test::Cluster->new('node_c');
-$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1);
-$node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef);
-$node_c->set_standby_mode();
-
-# Run pg_createsubscriber on node C (P -> S -> C)
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_c->data_dir,
-		'--publisher-server' => $node_s->connstr($db1),
-		'--socketdir' => $node_c->host,
-		'--subscriber-port' => $node_c->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'primary server is in recovery');
-
-# Check some unmet conditions on node P
-$node_p->append_conf(
-	'postgresql.conf', q{
-wal_level = replica
-max_replication_slots = 1
-max_wal_senders = 1
-max_worker_processes = 2
-});
-$node_p->restart;
-$node_s->stop;
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
-		'--database' => $db2,
-
-	],
-	'primary contains unmet conditions on node P');
-# Restore default settings here but only apply it after testing standby. Some
-# standby settings should not be a lower setting than on the primary.
-$node_p->append_conf(
-	'postgresql.conf', q{
-wal_level = logical
-max_replication_slots = 10
-max_wal_senders = 10
-max_worker_processes = 8
-});
-
-# Check some unmet conditions on node S
-$node_s->append_conf(
-	'postgresql.conf', q{
-max_replication_slots = 1
-max_logical_replication_workers = 1
-max_worker_processes = 2
-});
-command_fails(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'standby contains unmet conditions on node S');
-$node_s->append_conf(
-	'postgresql.conf', q{
-max_replication_slots = 10
-max_logical_replication_workers = 4
-max_worker_processes = 8
-});
-# Restore default settings on both servers
-$node_p->restart;
-
-# Create failover slot to test its removal
-my $fslotname = 'failover_slot';
-$node_p->safe_psql($db1,
-	"SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)"
-);
-$node_s->start;
-# Wait for the standby to catch up so that the standby is not lagging behind
-# the failover slot.
-$node_p->wait_for_replay_catchup($node_s);
-$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()");
-my $result = $node_s->safe_psql('postgres',
-	"SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary"
-);
-is($result, 'failover_slot', 'failover slot is synced');
-
-# Insert another row on node P and wait node S to catch up. We
-# intentionally performed this insert after syncing logical slot
-# as otherwise the local slot's (created during synchronization of
-# slot) xmin on standby could be ahead of the remote slot leading
-# to failure in synchronization.
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')");
-$node_p->wait_for_replay_catchup($node_s);
-
-# Create subscription to test its removal
-my $dummy_sub = 'regress_sub_dummy';
-$node_p->safe_psql($db1,
-	"CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)"
-);
-$node_p->wait_for_replay_catchup($node_s);
-$node_s->stop;
-
-# dry run mode on node S
-command_ok(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--publication' => 'pub1',
-		'--publication' => 'pub2',
-		'--subscription' => 'sub1',
-		'--subscription' => 'sub2',
-		'--database' => $db1,
-		'--database' => $db2,
-	],
-	'run pg_createsubscriber --dry-run on node S');
-
-# Check if node S is still a standby
-$node_s->start;
-is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'),
-	't', 'standby is in recovery');
-$node_s->stop;
-
-# pg_createsubscriber can run without --databases option
-command_ok(
-	[
-		'pg_createsubscriber',
-		'--verbose',
-		'--dry-run',
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--replication-slot' => 'replslot1',
-	],
-	'run pg_createsubscriber without --databases');
-
-# Run pg_createsubscriber on node S.  --verbose is used twice
-# to show more information.
-# In passing, also test the --enable-two-phase option
-command_ok(
-	[
-		'pg_createsubscriber',
-		'--verbose', '--verbose',
-		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
-		'--pgdata' => $node_s->data_dir,
-		'--publisher-server' => $node_p->connstr($db1),
-		'--socketdir' => $node_s->host,
-		'--subscriber-port' => $node_s->port,
-		'--publication' => 'pub1',
-		'--publication' => 'pub2',
-		'--replication-slot' => 'replslot1',
-		'--replication-slot' => 'replslot2',
-		'--database' => $db1,
-		'--database' => $db2,
-		'--enable-two-phase'
-	],
-	'run pg_createsubscriber on node S');
-
-# Confirm the physical replication slot has been removed
-$result = $node_p->safe_psql($db1,
-	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$slotname'"
-);
-is($result, qq(0),
-	'the physical replication slot used as primary_slot_name has been removed'
-);
-
-# Insert rows on P
-$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')");
-$node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')");
-
-# Start subscriber
-$node_s->start;
-
-# Verify that all subtwophase states are pending or enabled,
-# e.g. there are no subscriptions where subtwophase is disabled ('d')
-is( $node_s->safe_psql(
-		'postgres',
-		"SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'"
-	),
-	't',
-	'subscriptions are created with the two-phase option enabled');
-
-# Confirm the pre-existing subscription has been removed
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub'
-));
-is($result, qq(0), 'pre-existing subscription was dropped');
-
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
-
-# Confirm the failover slot has been removed
-$result = $node_s->safe_psql($db1,
-	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'"
-);
-is($result, qq(0), 'failover slot was removed');
-
-# Check result in database $db1
-$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
-is( $result, qq(first row
-second row
-third row),
-	"logical replication works in database $db1");
-
-# Check result in database $db2
-$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2');
-is($result, qq(row 1), "logical replication works in database $db2");
-
-# Different system identifier?
-my $sysid_p = $node_p->safe_psql('postgres',
-	'SELECT system_identifier FROM pg_control_system()');
-my $sysid_s = $node_s->safe_psql('postgres',
-	'SELECT system_identifier FROM pg_control_system()');
-ok($sysid_p != $sysid_s, 'system identifier was changed');
-
-# clean up
-$node_p->teardown_node;
-$node_s->teardown_node;
-$node_t->teardown_node;
-$node_f->teardown_node;
-
 done_testing();
diff --git a/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl b/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl
new file mode 100644
index 00000000000..a51c86ed9f3
--- /dev/null
+++ b/src/bin/pg_basebackup/t/041_covert_standby_to_subscriber.pl
@@ -0,0 +1,386 @@
+# Copyright (c) 2024-2025, PostgreSQL Global Development Group
+
+#
+# Test using a standby server as the subscriber.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Generate a database with a name made of a range of ASCII characters.
+# Extracted from 002_pg_upgrade.pl.
+sub generate_db
+{
+	my ($node, $prefix, $from_char, $to_char, $suffix) = @_;
+
+	my $dbname = $prefix;
+	for my $i ($from_char .. $to_char)
+	{
+		next if $i == 7 || $i == 10 || $i == 13;    # skip BEL, LF, and CR
+		$dbname = $dbname . sprintf('%c', $i);
+	}
+
+	$dbname .= $suffix;
+
+	# On Windows, older IPC::Run versions can mis-quote command line arguments
+	# containing double quote or backslash
+	$dbname =~ tr/\"\\//d if ($windows_os);
+
+	$node->command_ok(
+		[ 'createdb', $dbname ],
+		"created database with ASCII characters from $from_char to $to_char");
+
+	return $dbname;
+}
+
+#
+# Set up node P as primary
+my $node_p = PostgreSQL::Test::Cluster->new('node_p');
+my $pconnstr = $node_p->connstr;
+$node_p->init(allows_streaming => 'logical');
+# Disable autovacuum to avoid generating xid during stats update as otherwise
+# the new XID could then be replicated to standby at some random point making
+# slots at primary lag behind standby during slot sync.
+$node_p->append_conf('postgresql.conf', 'autovacuum = off');
+$node_p->start;
+
+# Set up node F as about-to-fail node
+# Force it to initialize a new cluster instead of copying a
+# previously initdb'd cluster. New cluster has a different system identifier so
+# we can test if the target cluster is a copy of the source cluster.
+my $node_f = PostgreSQL::Test::Cluster->new('node_f');
+$node_f->init(force_initdb => 1, allows_streaming => 'logical');
+
+# On node P
+# - create databases
+# - create test tables
+# - insert a row
+# - create a physical replication slot
+my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
+my $db2 = generate_db($node_p, 'regression', 46, 90, '');
+
+$node_p->safe_psql($db1, 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl2 (a text)');
+my $slotname = 'physical_slot';
+$node_p->safe_psql($db2,
+	"SELECT pg_create_physical_replication_slot('$slotname')");
+
+# Set up node S as standby linking to node P
+$node_p->backup('backup_1');
+my $node_s = PostgreSQL::Test::Cluster->new('node_s');
+$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_s->append_conf(
+	'postgresql.conf', qq[
+primary_slot_name = '$slotname'
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s->set_standby_mode();
+$node_s->start;
+
+# Set up node T as standby linking to node P then promote it
+my $node_t = PostgreSQL::Test::Cluster->new('node_t');
+$node_t->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_t->set_standby_mode();
+$node_t->start;
+$node_t->promote;
+$node_t->stop;
+
+# Run pg_createsubscriber on a promoted server
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_t->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_t->host,
+		'--subscriber-port' => $node_t->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'target server is not in recovery');
+
+# Run pg_createsubscriber when standby is running
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'standby is up and running');
+
+# Run pg_createsubscriber on about-to-fail node F
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_f->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_f->host,
+		'--subscriber-port' => $node_f->port,
+		'--database' => $db1,
+		'--database' => $db2
+	],
+	'subscriber data directory is not a copy of the source database cluster');
+
+# Set up node C as standby linking to node S
+$node_s->backup('backup_2');
+my $node_c = PostgreSQL::Test::Cluster->new('node_c');
+$node_c->init_from_backup($node_s, 'backup_2', has_streaming => 1);
+$node_c->adjust_conf('postgresql.conf', 'primary_slot_name', undef);
+$node_c->set_standby_mode();
+
+# Run pg_createsubscriber on node C (P -> S -> C)
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_c->data_dir,
+		'--publisher-server' => $node_s->connstr($db1),
+		'--socketdir' => $node_c->host,
+		'--subscriber-port' => $node_c->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'primary server is in recovery');
+
+# Check some unmet conditions on node P
+$node_p->append_conf(
+	'postgresql.conf', q{
+wal_level = replica
+max_replication_slots = 1
+max_wal_senders = 1
+max_worker_processes = 2
+});
+$node_p->restart;
+$node_s->stop;
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--database' => $db2,
+
+	],
+	'primary contains unmet conditions on node P');
+# Restore default settings here but only apply it after testing standby. Some
+# standby settings should not be a lower setting than on the primary.
+$node_p->append_conf(
+	'postgresql.conf', q{
+wal_level = logical
+max_replication_slots = 10
+max_wal_senders = 10
+max_worker_processes = 8
+});
+
+# Check some unmet conditions on node S
+$node_s->append_conf(
+	'postgresql.conf', q{
+max_replication_slots = 1
+max_logical_replication_workers = 1
+max_worker_processes = 2
+});
+command_fails(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'standby contains unmet conditions on node S');
+$node_s->append_conf(
+	'postgresql.conf', q{
+max_replication_slots = 10
+max_logical_replication_workers = 4
+max_worker_processes = 8
+});
+# Restore default settings on both servers
+$node_p->restart;
+
+# Create failover slot to test its removal
+my $fslotname = 'failover_slot';
+$node_p->safe_psql($db1,
+	"SELECT pg_create_logical_replication_slot('$fslotname', 'pgoutput', false, false, true)"
+);
+$node_s->start;
+# Wait for the standby to catch up so that the standby is not lagging behind
+# the failover slot.
+$node_p->wait_for_replay_catchup($node_s);
+$node_s->safe_psql('postgres', "SELECT pg_sync_replication_slots()");
+my $result = $node_s->safe_psql('postgres',
+	"SELECT slot_name FROM pg_replication_slots WHERE slot_name = '$fslotname' AND synced AND NOT temporary"
+);
+is($result, 'failover_slot', 'failover slot is synced');
+
+# Insert another row on node P and wait node S to catch up. We
+# intentionally performed this insert after syncing logical slot
+# as otherwise the local slot's (created during synchronization of
+# slot) xmin on standby could be ahead of the remote slot leading
+# to failure in synchronization.
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('second row')");
+$node_p->wait_for_replay_catchup($node_s);
+
+# Create subscription to test its removal
+my $dummy_sub = 'regress_sub_dummy';
+$node_p->safe_psql($db1,
+	"CREATE SUBSCRIPTION $dummy_sub CONNECTION 'dbname=dummy' PUBLICATION pub_dummy WITH (connect=false)"
+);
+$node_p->wait_for_replay_catchup($node_s);
+$node_s->stop;
+
+# dry run mode on node S
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--publication' => 'pub2',
+		'--subscription' => 'sub1',
+		'--subscription' => 'sub2',
+		'--database' => $db1,
+		'--database' => $db2,
+	],
+	'run pg_createsubscriber --dry-run on node S');
+
+# Check if node S is still a standby
+$node_s->start;
+is($node_s->safe_psql('postgres', 'SELECT pg_catalog.pg_is_in_recovery()'),
+	't', 'standby is in recovery');
+$node_s->stop;
+
+# pg_createsubscriber can run without --databases option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+	],
+	'run pg_createsubscriber without --databases');
+
+# Run pg_createsubscriber on node S.  --verbose is used twice
+# to show more information.
+# In passing, also test the --enable-two-phase option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose', '--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--publication' => 'pub1',
+		'--publication' => 'pub2',
+		'--replication-slot' => 'replslot1',
+		'--replication-slot' => 'replslot2',
+		'--database' => $db1,
+		'--database' => $db2,
+		'--enable-two-phase'
+	],
+	'run pg_createsubscriber on node S');
+
+# Confirm the physical replication slot has been removed
+$result = $node_p->safe_psql($db1,
+	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$slotname'"
+);
+is($result, qq(0),
+	'the physical replication slot used as primary_slot_name has been removed'
+);
+
+# Insert rows on P
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('third row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl2 VALUES('row 1')");
+
+# Start subscriber
+$node_s->start;
+
+# Verify that all subtwophase states are pending or enabled,
+# e.g. there are no subscriptions where subtwophase is disabled ('d')
+is( $node_s->safe_psql(
+		'postgres',
+		"SELECT count(1) = 0 FROM pg_subscription WHERE subtwophasestate = 'd'"
+	),
+	't',
+	'subscriptions are created with the two-phase option enabled');
+
+# Confirm the pre-existing subscription has been removed
+$result = $node_s->safe_psql(
+	'postgres', qq(
+	SELECT count(*) FROM pg_subscription WHERE subname = '$dummy_sub'
+));
+is($result, qq(0), 'pre-existing subscription was dropped');
+
+# Get subscription names
+$result = $node_s->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
+$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+
+# Confirm the failover slot has been removed
+$result = $node_s->safe_psql($db1,
+	"SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$fslotname'"
+);
+is($result, qq(0), 'failover slot was removed');
+
+# Check result in database $db1
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2
+$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl2');
+is($result, qq(row 1), "logical replication works in database $db2");
+
+# Different system identifier?
+my $sysid_p = $node_p->safe_psql('postgres',
+	'SELECT system_identifier FROM pg_control_system()');
+my $sysid_s = $node_s->safe_psql('postgres',
+	'SELECT system_identifier FROM pg_control_system()');
+ok($sysid_p != $sysid_s, 'system identifier was changed');
+
+# clean up
+$node_p->teardown_node;
+$node_s->teardown_node;
+$node_t->teardown_node;
+$node_f->teardown_node;
+
+done_testing();
diff --git a/src/bin/pg_basebackup/t/042_all_option.pl b/src/bin/pg_basebackup/t/042_all_option.pl
new file mode 100644
index 00000000000..18e74c294b9
--- /dev/null
+++ b/src/bin/pg_basebackup/t/042_all_option.pl
@@ -0,0 +1,254 @@
+# Copyright (c) 2024-2025, PostgreSQL Global Development Group
+
+#
+# Test using a standby server as the subscriber.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Generate a database with a name made of a range of ASCII characters.
+# Extracted from 002_pg_upgrade.pl.
+sub generate_db
+{
+	my ($node, $prefix, $from_char, $to_char, $suffix) = @_;
+
+	my $dbname = $prefix;
+	for my $i ($from_char .. $to_char)
+	{
+		next if $i == 7 || $i == 10 || $i == 13;    # skip BEL, LF, and CR
+		$dbname = $dbname . sprintf('%c', $i);
+	}
+
+	$dbname .= $suffix;
+
+	# On Windows, older IPC::Run versions can mis-quote command line arguments
+	# containing double quote or backslash
+	$dbname =~ tr/\"\\//d if ($windows_os);
+
+	$node->command_ok(
+		[ 'createdb', $dbname ],
+		"created database with ASCII characters from $from_char to $to_char");
+
+	return $dbname;
+}
+
+#
+# Set up node P as primary
+my $node_p = PostgreSQL::Test::Cluster->new('node_p');
+my $pconnstr = $node_p->connstr;
+$node_p->init(allows_streaming => 'logical');
+# Disable autovacuum to avoid generating xid during stats update as otherwise
+# the new XID could then be replicated to standby at some random point making
+# slots at primary lag behind standby during slot sync.
+$node_p->append_conf('postgresql.conf', 'autovacuum = off');
+$node_p->start;
+
+# On node P
+# - create databases
+# - create test tables
+# - insert a row
+my $db1 = generate_db($node_p, 'regression\\"\\', 1, 45, '\\\\"\\\\\\');
+my $db2 = generate_db($node_p, 'regression', 46, 90, '');
+
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');
+
+# Set up node S as standby linking to node P
+$node_p->backup('backup_1');
+my $node_s = PostgreSQL::Test::Cluster->new('node_s');
+$node_s->init_from_backup($node_p, 'backup_1', has_streaming => 1);
+$node_s->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+hot_standby_feedback = on
+]);
+$node_s->set_standby_mode();
+
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--all' and '--database' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all without --dry-run');
+
+# run pg_createsubscriber with '--all' and '--publication' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' and '--replication-slot' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--replication-slot' => 'replslot1',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--all' and '--subscription' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
+# run pg_createsubscriber with '--all' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all without --dry-run');
+
+$node_s->start;
+
+# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both
+# $db1 and $db2 must be escaped to pass the safe_psql(), but it is difficult.
+# Thus, we define a cursor, obtain a dbname from the instance and compere one
+# by one.
+my @user_dbs = ('postgres', $db1, $db2);
+
+my $bgconn = $node_s->background_psql('postgres');
+$bgconn->query_safe(
+	qq[
+	BEGIN;
+	DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database
+	WHERE subdbid = pg_database.oid and datistemplate = 'f'
+	ORDER BY pg_database.oid;
+]);
+
+# Fetch from the cursor three times and confirm the existence of the
+# subscription on $dbname
+foreach my $dbname (@user_dbs)
+{
+	my $result = $bgconn->query_safe("FETCH cursor;");
+
+	is($result, $dbname, "subscription is created on $dbname");
+}
+$bgconn->quit;
+
+# Verify template databases do not have subscriptions
+my $result = $node_s->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')");
+
+# Get subscription names
+$result = $node_s->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
+$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+$node_s->wait_for_subscription_sync($node_p, $subnames[2]);
+
+# Check result in database 'postgres' of node S
+$result = $node_s->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node S
+$result = $node_s->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2 of node S
+$result = $node_s->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+
+# clean up
+$node_p->teardown_node;
+$node_s->teardown_node;
+
+done_testing();
-- 
2.41.0.windows.3

#66vignesh C
vignesh21@gmail.com
In reply to: Shubham Khanna (#65)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, 17 Mar 2025 at 11:28, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Fri, Mar 14, 2025 at 5:43 PM Nisha Moond <nisha.moond412@gmail.com> wrote:

Fixed.

The attached patch contains the suggested changes.

I feel like we're trying to address two separate tasks in this thread:
a) Enhancing pg_createsubscriber to automatically retrieve databases
when none is provided. b) Refactoring all pg_createsubscriber tests.

I suggest we keep this thread focused solely on retrieving all
databases and start a new thread for test refactoring. This will allow
us to tackle each task separately and ensure a cleaner commit.

Regards,
Vignesh

#67Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: vignesh C (#66)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 17, 2025 at 4:02 PM vignesh C <vignesh21@gmail.com> wrote:

On Mon, 17 Mar 2025 at 11:28, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Fri, Mar 14, 2025 at 5:43 PM Nisha Moond <nisha.moond412@gmail.com> wrote:

Fixed.

The attached patch contains the suggested changes.

I feel like we're trying to address two separate tasks in this thread:
a) Enhancing pg_createsubscriber to automatically retrieve databases
when none is provided. b) Refactoring all pg_createsubscriber tests.

I suggest we keep this thread focused solely on retrieving all
databases and start a new thread for test refactoring. This will allow
us to tackle each task separately and ensure a cleaner commit.

I was expecting that the argument validation tests will go in one test
- existing as well as the ones for --all. But that's not how the patch
is splitting them. It has split only the existing test. I am fine if
we add a new test for --all option as the [patch does and leave the
existing test as is. Cramming everything in one test makes it
unmaintainable, though.

--
Best Wishes,
Ashutosh Bapat

#68vignesh C
vignesh21@gmail.com
In reply to: Ashutosh Bapat (#67)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, 17 Mar 2025 at 16:51, Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Mon, Mar 17, 2025 at 4:02 PM vignesh C <vignesh21@gmail.com> wrote:

On Mon, 17 Mar 2025 at 11:28, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Fri, Mar 14, 2025 at 5:43 PM Nisha Moond <nisha.moond412@gmail.com> wrote:

Fixed.

The attached patch contains the suggested changes.

I feel like we're trying to address two separate tasks in this thread:
a) Enhancing pg_createsubscriber to automatically retrieve databases
when none is provided. b) Refactoring all pg_createsubscriber tests.

I suggest we keep this thread focused solely on retrieving all
databases and start a new thread for test refactoring. This will allow
us to tackle each task separately and ensure a cleaner commit.

I was expecting that the argument validation tests will go in one test
- existing as well as the ones for --all. But that's not how the patch
is splitting them. It has split only the existing test. I am fine if
we add a new test for --all option as the [patch does and leave the
existing test as is. Cramming everything in one test makes it
unmaintainable, though.

For this patch, let's add an additional test case to
040_pg_createsubscriber.pl and aim to commit it soon, as the rest of
the changes look good. I agree that the test split you suggested is
necessary, but let's handle that in a separate thread.

Regards,
Vignesh

#69Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: vignesh C (#68)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 18, 2025 at 4:25 PM vignesh C <vignesh21@gmail.com> wrote:

On Mon, 17 Mar 2025 at 16:51, Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Mon, Mar 17, 2025 at 4:02 PM vignesh C <vignesh21@gmail.com> wrote:

On Mon, 17 Mar 2025 at 11:28, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Fri, Mar 14, 2025 at 5:43 PM Nisha Moond <nisha.moond412@gmail.com> wrote:

Fixed.

The attached patch contains the suggested changes.

I feel like we're trying to address two separate tasks in this thread:
a) Enhancing pg_createsubscriber to automatically retrieve databases
when none is provided. b) Refactoring all pg_createsubscriber tests.

I suggest we keep this thread focused solely on retrieving all
databases and start a new thread for test refactoring. This will allow
us to tackle each task separately and ensure a cleaner commit.

I was expecting that the argument validation tests will go in one test
- existing as well as the ones for --all. But that's not how the patch
is splitting them. It has split only the existing test. I am fine if
we add a new test for --all option as the [patch does and leave the
existing test as is. Cramming everything in one test makes it
unmaintainable, though.

For this patch, let's add an additional test case to
040_pg_createsubscriber.pl and aim to commit it soon, as the rest of
the changes look good. I agree that the test split you suggested is
necessary, but let's handle that in a separate thread.

I am fine with it. I am just worried about the resultant test being
unreadable and patch hard to review. That's how the first patch was
written.

--
Best Wishes,
Ashutosh Bapat

#70Shubham Khanna
khannashubham1197@gmail.com
In reply to: vignesh C (#68)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 18, 2025 at 4:25 PM vignesh C <vignesh21@gmail.com> wrote:

On Mon, 17 Mar 2025 at 16:51, Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Mon, Mar 17, 2025 at 4:02 PM vignesh C <vignesh21@gmail.com> wrote:

On Mon, 17 Mar 2025 at 11:28, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Fri, Mar 14, 2025 at 5:43 PM Nisha Moond <nisha.moond412@gmail.com> wrote:

Fixed.

The attached patch contains the suggested changes.

I feel like we're trying to address two separate tasks in this thread:
a) Enhancing pg_createsubscriber to automatically retrieve databases
when none is provided. b) Refactoring all pg_createsubscriber tests.

I suggest we keep this thread focused solely on retrieving all
databases and start a new thread for test refactoring. This will allow
us to tackle each task separately and ensure a cleaner commit.

I was expecting that the argument validation tests will go in one test
- existing as well as the ones for --all. But that's not how the patch
is splitting them. It has split only the existing test. I am fine if
we add a new test for --all option as the [patch does and leave the
existing test as is. Cramming everything in one test makes it
unmaintainable, though.

For this patch, let's add an additional test case to
040_pg_createsubscriber.pl and aim to commit it soon, as the rest of
the changes look good. I agree that the test split you suggested is
necessary, but let's handle that in a separate thread.

I have added an additional test case to 040_pg_createsubscriber.pl as suggested.

The attached patch contains the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v14-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v14-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From 1e83f955683c3b6085d9a233f1e7433714977516 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 28 Feb 2025 17:29:30 +0530
Subject: [PATCH v14] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 ++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 ++++++++-
 .../t/040_pg_createsubscriber.pl              | 195 ++++++++++++++++++
 3 files changed, 329 insertions(+), 10 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index b4b996236e4..7376c61f3eb 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,21 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,9 +130,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.
       </para>
      </listitem>
@@ -230,7 +267,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -246,7 +284,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -261,7 +300,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 6baf92e8024..f918a0e33b8 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -44,6 +44,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 /* per-database publication/subscription info */
@@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -236,6 +238,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1904,11 +1907,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1978,6 +2030,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1995,11 +2048,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2087,6 +2143,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2140,14 +2218,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c35fa108ce3..dca5f95abbc 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,104 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
+# run pg_createsubscriber with '--all' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option
@@ -459,10 +557,107 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both
+# $db1 and $db2 must be escaped to pass the safe_psql(), but it is difficult.
+# Thus, we define a cursor, obtain a dbname from the instance and compere one
+# by one.
+my @user_dbs = ('postgres', $db1, $db2);
+
+my $bgconn = $node_u->background_psql('postgres');
+$bgconn->query_safe(
+	qq[
+	BEGIN;
+	DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription, pg_database
+	WHERE subdbid = pg_database.oid and datistemplate = 'f'
+	ORDER BY pg_database.oid;
+]);
+
+# Fetch from the cursor three times and confirm the existence of the
+# subscription on $dbname
+foreach my $dbname (@user_dbs)
+{
+	my $result = $bgconn->query_safe("FETCH cursor;");
+
+	is($result, $dbname, "subscription is created on $dbname");
+}
+$bgconn->quit;
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')");
+
+# Get subscription names
+$result = $node_u->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.34.1

#71vignesh C
vignesh21@gmail.com
In reply to: Shubham Khanna (#70)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, 18 Mar 2025 at 17:34, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I have added an additional test case to 040_pg_createsubscriber.pl as suggested.

The attached patch contains the suggested changes.

How about we change the below code:
+# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both
+# $db1 and $db2 must be escaped to pass the safe_psql(), but it is difficult.
+# Thus, we define a cursor, obtain a dbname from the instance and compere one
+# by one.
+my @user_dbs = ('postgres', $db1, $db2);
+
+my $bgconn = $node_u->background_psql('postgres');
+$bgconn->query_safe(
+       qq[
+       BEGIN;
+       DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription,
pg_database
+       WHERE subdbid = pg_database.oid and datistemplate = 'f'
+       ORDER BY pg_database.oid;
+]);
+
+# Fetch from the cursor three times and confirm the existence of the
+# subscription on $dbname
+foreach my $dbname (@user_dbs)
+{
+       my $result = $bgconn->query_safe("FETCH cursor;");
+
+       is($result, $dbname, "subscription is created on $dbname");
+}

like:
$result = $node_u->safe_psql($db1, 'SELECT datname FROM
pg_subscription, pg_database WHERE subdbid = pg_database.oid and
datistemplate = \'f\' ORDER BY pg_database.oid');
is($result, "postgres
$db1
$db2", 'subscription is created on the required databases');

I felt this might simplify your verification logic.

Regards,
Vignesh

#72Shubham Khanna
khannashubham1197@gmail.com
In reply to: vignesh C (#71)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Mar 19, 2025 at 11:17 AM vignesh C <vignesh21@gmail.com> wrote:

On Tue, 18 Mar 2025 at 17:34, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I have added an additional test case to 040_pg_createsubscriber.pl as suggested.

The attached patch contains the suggested changes.

How about we change the below code:
+# Verify that user databases (postgres, $db1, $db2) got subscriptions. Both
+# $db1 and $db2 must be escaped to pass the safe_psql(), but it is difficult.
+# Thus, we define a cursor, obtain a dbname from the instance and compere one
+# by one.
+my @user_dbs = ('postgres', $db1, $db2);
+
+my $bgconn = $node_u->background_psql('postgres');
+$bgconn->query_safe(
+       qq[
+       BEGIN;
+       DECLARE cursor CURSOR FOR SELECT datname FROM pg_subscription,
pg_database
+       WHERE subdbid = pg_database.oid and datistemplate = 'f'
+       ORDER BY pg_database.oid;
+]);
+
+# Fetch from the cursor three times and confirm the existence of the
+# subscription on $dbname
+foreach my $dbname (@user_dbs)
+{
+       my $result = $bgconn->query_safe("FETCH cursor;");
+
+       is($result, $dbname, "subscription is created on $dbname");
+}

like:
$result = $node_u->safe_psql($db1, 'SELECT datname FROM
pg_subscription, pg_database WHERE subdbid = pg_database.oid and
datistemplate = \'f\' ORDER BY pg_database.oid');
is($result, "postgres
$db1
$db2", 'subscription is created on the required databases');

I felt this might simplify your verification logic.

I agree with you on this; switching to a single query with safe_psql()
will indeed simplify the test and make the verification logic cleaner.

The attached patch contains the suggested change.

Thanks and regards,
Shubham Khanna.

Attachments:

v15-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v15-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From a0b82406b805cb89037bcf0e446e02d8af16447b Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 28 Feb 2025 17:29:30 +0530
Subject: [PATCH v15] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 ++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 ++++++++-
 .../t/040_pg_createsubscriber.pl              | 180 ++++++++++++++++++
 3 files changed, 314 insertions(+), 10 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index b4b996236e4..7376c61f3eb 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,21 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,9 +130,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.
       </para>
      </listitem>
@@ -230,7 +267,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -246,7 +284,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -261,7 +300,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 6baf92e8024..f918a0e33b8 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -44,6 +44,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 /* per-database publication/subscription info */
@@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -236,6 +238,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1904,11 +1907,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1978,6 +2030,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1995,11 +2048,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2087,6 +2143,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2140,14 +2218,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c35fa108ce3..d109f8bc493 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,104 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
+# run pg_createsubscriber with '--all' option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option
@@ -459,10 +557,92 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1, $db2) got subscriptions.
+$result = $node_u->safe_psql(
+	$db1,
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1
+$db2", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')");
+
+# Get subscription names
+$result = $node_u->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.34.1

#73vignesh C
vignesh21@gmail.com
In reply to: Shubham Khanna (#72)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, 19 Mar 2025 at 11:44, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I agree with you on this; switching to a single query with safe_psql()
will indeed simplify the test and make the verification logic cleaner.

The attached patch contains the suggested change.

Few comments:
1) I felt we are not doing anything specific to dry-run, so no need to
have this test case:
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--dry-run',
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--all',
+               '--database' => $db1,
+       ],
+       qr/--database cannot be used with --all/,
+       'fail if --database is used with --all');
+
2) Similarly this too:
+# run pg_createsubscriber with '--all' option
+command_ok(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--dry-run',
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--all',
+       ],
+       'run pg_createsubscriber with --all');
+
3) We have run other pg_createsubscriber success tests with
recovery-timeout specified, so I suggest modifying this too similarly:
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--pgdata' => $node_u->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_u->host,
+               '--subscriber-port' => $node_u->port,
+               '--all',
+       ],
+       'run pg_createsubscriber with --all');
+
+$node_u->start;

4) You can try to see how much extra time it takes to run the tests,
if takes more time, then you can think of the following changes a) few
of the option validation test cases can be removed, we can have just
one combination of all with either of
publication/subscription/replication slot b) for the last test added,
you can drop one of the dbs and verify the subscriptions are created
in 2 dbs as the code flow is the same for all databases.

Regards,
Vignesh

#74Shubham Khanna
khannashubham1197@gmail.com
In reply to: vignesh C (#73)
2 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Mar 19, 2025 at 2:29 PM vignesh C <vignesh21@gmail.com> wrote:

On Wed, 19 Mar 2025 at 11:44, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I agree with you on this; switching to a single query with safe_psql()
will indeed simplify the test and make the verification logic cleaner.

The attached patch contains the suggested change.

Few comments:
1) I felt we are not doing anything specific to dry-run, so no need to
have this test case:
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--dry-run',
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--all',
+               '--database' => $db1,
+       ],
+       qr/--database cannot be used with --all/,
+       'fail if --database is used with --all');
+

Fixed.

2) Similarly this too:
+# run pg_createsubscriber with '--all' option
+command_ok(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--dry-run',
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--all',
+       ],
+       'run pg_createsubscriber with --all');
+

Fixed.

3) We have run other pg_createsubscriber success tests with
recovery-timeout specified, so I suggest modifying this too similarly:
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--pgdata' => $node_u->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_u->host,
+               '--subscriber-port' => $node_u->port,
+               '--all',
+       ],
+       'run pg_createsubscriber with --all');
+
+$node_u->start;

Fixed.

4) You can try to see how much extra time it takes to run the tests,
if takes more time, then you can think of the following changes a) few
of the option validation test cases can be removed, we can have just
one combination of all with either of
publication/subscription/replication slot b) for the last test added,
you can drop one of the dbs and verify the subscriptions are created
in 2 dbs as the code flow is the same for all databases.

I have created two patches, v16-0001 and v16-0002, to address the
performance issue. I conducted performance testing, and here are the
results:
- The difference in execution time between HEAD and the v15 patch was 53.2%.
- After removing the suggested test cases, the difference reduced to
36.43%, showing a significant improvement.

The optimizations included:
- Reducing redundant option validation test cases by retaining only
one combination of publication/subscription/replication slot.
- Dropping one of the databases in the last test case and verifying
that subscriptions are created correctly in the remaining two
databases, as the code flow remains consistent across databases.

The attached patches contain the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v16-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v16-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From ce94be3de9566716a9fb2f3db4721319c8f2dba1 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 28 Feb 2025 17:29:30 +0530
Subject: [PATCH v16 1/2] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  52 +++++-
 src/bin/pg_basebackup/pg_createsubscriber.c   |  92 ++++++++++-
 .../t/040_pg_createsubscriber.pl              | 150 ++++++++++++++++++
 3 files changed, 284 insertions(+), 10 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index b4b996236e4..7376c61f3eb 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
@@ -87,6 +108,21 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.
+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,9 +130,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.
       </para>
      </listitem>
@@ -230,7 +267,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -246,7 +284,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -261,7 +300,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 6baf92e8024..f918a0e33b8 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -44,6 +44,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 };
 
 /* per-database publication/subscription info */
@@ -118,6 +119,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -236,6 +238,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1904,11 +1907,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -1978,6 +2030,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -1995,11 +2048,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2087,6 +2143,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2140,14 +2218,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c35fa108ce3..1fe158995ab 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,6 +371,73 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option
@@ -459,10 +526,93 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1, $db2) got subscriptions.
+$result = $node_u->safe_psql(
+	$db1,
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1
+$db2", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')");
+$node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')");
+
+# Get subscription names
+$result = $node_u->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+	"logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

v16-0002-Additional-test-cases.patchapplication/octet-stream; name=v16-0002-Additional-test-cases.patchDownload
From cd985dd1da8fb15223accad8978a6e1cf1989625 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 09:39:07 +0530
Subject: [PATCH v16 2/2] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 45 ++++++++-----------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 1fe158995ab..b92e6d34727 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -371,41 +371,41 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
-# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
-# and verify the failure
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
 command_fails_like(
 	[
 		'pg_createsubscriber',
 		'--verbose',
+		'--dry-run',
 		'--pgdata' => $node_s->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
-		'--database' => $db1,
 		'--all',
+		'--database' => $db1,
 	],
 	qr/--database cannot be used with --all/,
 	'fail if --database is used with --all');
 
-# run pg_createsubscriber with '--publication' and '--all' and verify
-# the failure
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
 command_fails_like(
 	[
 		'pg_createsubscriber',
 		'--verbose',
-		'--dry-run',
 		'--pgdata' => $node_s->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
 		'--all',
-		'--publication' => 'pub1',
 	],
-	qr/--publication cannot be used with --all/,
-	'fail if --publication is used with --all');
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
 
-# run pg_createsubscriber with '--replication-slot' and '--all' and
-# verify the failure
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
 command_fails_like(
 	[
 		'pg_createsubscriber',
@@ -415,28 +415,26 @@ command_fails_like(
 		'--publisher-server' => $node_p->connstr($db1),
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
-		'--replication-slot' => 'replslot1',
 		'--all',
+		'--publication' => 'pub1',
 	],
-	qr/--replication-slot cannot be used with --all/,
-	'fail if --replication-slot is used with --all');
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
 
-# run pg_createsubscriber with '--subscription' and '--all' and
-# verify the failure
-command_fails_like(
+# run pg_createsubscriber with '--all' option
+command_ok(
 	[
 		'pg_createsubscriber',
 		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
 		'--dry-run',
 		'--pgdata' => $node_s->data_dir,
 		'--publisher-server' => $node_p->connstr($db1),
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
 		'--all',
-		'--subscription' => 'sub1',
 	],
-	qr/--subscription cannot be used with --all/,
-	'fail if --subscription is used with --all');
+	'run pg_createsubscriber with --all');
 
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
@@ -527,7 +525,6 @@ my $sysid_s = $node_s->safe_psql('postgres',
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
 # On node P create test tables
-$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
 $node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
 $node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
 $node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');
@@ -578,7 +575,6 @@ is($result, '0', 'subscription is not created on template databases');
 
 # Verify logical replication works for all databases
 # Insert rows on node P
-$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
 $node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('second row')");
 $node_p->safe_psql($db2, "INSERT INTO tbl3 VALUES('first row')");
 
@@ -592,11 +588,6 @@ my @subnames1 = split("\n", $result);
 # Wait subscriber to catch up
 $node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
 $node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
-$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
-
-# Check result in database 'postgres' of node U
-$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
-is($result, qq(first row), "logical replication works in database postgres");
 
 # Check result in database $db1 of node U
 $result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
-- 
2.41.0.windows.3

#75vignesh C
vignesh21@gmail.com
In reply to: Shubham Khanna (#74)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, 20 Mar 2025 at 10:25, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I have created two patches, v16-0001 and v16-0002, to address the
performance issue. I conducted performance testing, and here are the
results:
- The difference in execution time between HEAD and the v15 patch was 53.2%.
- After removing the suggested test cases, the difference reduced to
36.43%, showing a significant improvement.

It is still taking quite a while, can we compare with the following
too see how much extra time it takes:
a) remove insert and select verification for the all database
successful tests as all of these are logical replication verification
which is extensively tested b) remove command_fails_like failure tests
c) remove both of above and see.

Regards,
Vignesh

#76Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#74)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 10:25 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.

I have started reviewing the patches again. Here are some review comments

    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.

In this construct "all" goes with "source server" but it should go
with "non-template database". How about, "For every non-template
database on the source server, create one subscription on the target
server in the database with the same name."?

+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

... used along with ...

also comma before or .

We should also mention that generated names of replication slots,
publications and subscriptions are used when using this option.

+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable
class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable
class="parameter">dbname</replaceable></option></term>
@@ -94,9 +130,10 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with
<option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
        <option>-P</option> option, an error will be reported.

We have to mention -a as well here. Something like "If the database
name is not specified in either the <option>-d</option> option or
<option>-P</option> option, and <option>-a</option> is not provided,
an error will be reported."

+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--database cannot be used with --all/,
+ 'fail if --database is used with --all');

Why does only this test not use --dry-run, but all other such tests
use --dry-run? Either they should all use it or not use it.

+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--publication' => 'pub1',
+ ],
+ qr/--publication cannot be used with --all/,
+ 'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--replication-slot' => 'replslot1',
+ '--all',
+ ],
+ qr/--replication-slot cannot be used with --all/,
+ 'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--subscription' => 'sub1',
+ ],
+ qr/--subscription cannot be used with --all/,
+ 'fail if --subscription is used with --all');
+

Just to cover all combinations we need to test both scenarios. where
--all comes before an incompatible option and vice versa.

# Run pg_createsubscriber on node S. --verbose is used twice
# to show more information.
# In passing, also test the --enable-two-phase option
@@ -459,10 +526,93 @@ my $sysid_s = $node_s->safe_psql('postgres',
'SELECT system_identifier FROM pg_control_system()');
ok($sysid_p != $sysid_s, 'system identifier was changed');

+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');

$db1 and $db2 already have tables. We can avoid creating new ones here.

+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+

This function just makes sure that initial sync is done. But it
doesn't wait for replication to catch up with current state of the
publisher. It's the latter which would make sure that the last INSERTS
are visible. We should be using wait_for_slot_catchup() on the
publisher.

+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+ "logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+

These tests may not always pass if we use wait_for_slot_catchup above.

--
Best Wishes,
Ashutosh Bapat

#77Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: vignesh C (#75)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 3:41 PM vignesh C <vignesh21@gmail.com> wrote:

On Thu, 20 Mar 2025 at 10:25, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I have created two patches, v16-0001 and v16-0002, to address the
performance issue. I conducted performance testing, and here are the
results:
- The difference in execution time between HEAD and the v15 patch was 53.2%.
- After removing the suggested test cases, the difference reduced to
36.43%, showing a significant improvement.

It is still taking quite a while, can we compare with the following
too see how much extra time it takes:
a) remove insert and select verification for the all database
successful tests as all of these are logical replication verification
which is extensively tested

If the intent is to remove those, I don't think it's a good idea.
Other tests may be testing replication but they don't test the setup
created by --all, which consists of objects generated with names
crafted by utility.

Said that we should also test the behaviour when such objects already
exist. But I guess randomness in generated names is hard to control.

--
Best Wishes,
Ashutosh Bapat

#78Amit Kapila
amit.kapila16@gmail.com
In reply to: Ashutosh Bapat (#76)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 4:53 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 10:25 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--database cannot be used with --all/,
+ 'fail if --database is used with --all');

Why does only this test not use --dry-run, but all other such tests
use --dry-run? Either they should all use it or not use it.

I think this patch is adding a lot of extra tests which I don't think
are required. We should have one positive test in --dry-run mode
similar to ('run pg_createsubscriber without --databases') and
probably verify in the LOG that it has expanded commands for all
databases. Also, for negative tests, one or two tests are sufficient
instead of testing all possible combinations. I don't expect this new
option to add a noticeable testing time.

*  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>

Most of this is unrelated to this patch. I suggest making a top-up
patch, we can commit it separately.

--
With Regards,
Amit Kapila.

#79Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#76)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 4:53 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 10:25 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.

I have started reviewing the patches again. Here are some review comments

<variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.

In this construct "all" goes with "source server" but it should go
with "non-template database". How about, "For every non-template
database on the source server, create one subscription on the target
server in the database with the same name."?

Fixed.

+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

... used along with ...

also comma before or .

We should also mention that generated names of replication slots,
publications and subscriptions are used when using this option.

+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-d <replaceable
class="parameter">dbname</replaceable></option></term>
<term><option>--database=<replaceable
class="parameter">dbname</replaceable></option></term>

Fixed.

@@ -94,9 +130,10 @@ PostgreSQL documentation
<para>
The name of the database in which to create a subscription.  Multiple
databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with
<option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
<option>-P</option> option, an error will be reported.

We have to mention -a as well here. Something like "If the database
name is not specified in either the <option>-d</option> option or
<option>-P</option> option, and <option>-a</option> is not provided,
an error will be reported."

Fixed.

+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--database cannot be used with --all/,
+ 'fail if --database is used with --all');

Why does only this test not use --dry-run, but all other such tests
use --dry-run? Either they should all use it or not use it.

+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--publication' => 'pub1',
+ ],
+ qr/--publication cannot be used with --all/,
+ 'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--replication-slot' => 'replslot1',
+ '--all',
+ ],
+ qr/--replication-slot cannot be used with --all/,
+ 'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--subscription' => 'sub1',
+ ],
+ qr/--subscription cannot be used with --all/,
+ 'fail if --subscription is used with --all');
+

Just to cover all combinations we need to test both scenarios. where
--all comes before an incompatible option and vice versa.

# Run pg_createsubscriber on node S. --verbose is used twice
# to show more information.
# In passing, also test the --enable-two-phase option
@@ -459,10 +526,93 @@ my $sysid_s = $node_s->safe_psql('postgres',
'SELECT system_identifier FROM pg_control_system()');
ok($sysid_p != $sysid_s, 'system identifier was changed');

+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');

$db1 and $db2 already have tables. We can avoid creating new ones here.

Fixed.

+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+

This function just makes sure that initial sync is done. But it
doesn't wait for replication to catch up with current state of the
publisher. It's the latter which would make sure that the last INSERTS
are visible. We should be using wait_for_slot_catchup() on the
publisher.

+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+ "logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+

These tests may not always pass if we use wait_for_slot_catchup above.

During the recent testing, I observed that the tests were failing when
using wait_for_slot_catchup(). To address this, I reverted to using
wait_for_subscription_sync(), which was employed previously and has
proven to be more reliable in ensuring test stability.
Please let me know if there are any additional adjustments you would
suggest or if you would like me to investigate further into
wait_for_slot_catchup().

I have created a separate patch for the synopsis of '--all' option as
suggested by Amit at [1]/messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com. The attached patch contains the suggested
changes.

[1]: /messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

Attachments:

v17-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v17-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From aa09f2e978ec5c713e3b4e1716618a728176ed7b Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v17] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     | 35 +++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 92 ++++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              | 69 ++++++++++++++
 3 files changed, 185 insertions(+), 11 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 380d0b1c35c..3220b0cfa7b 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
+       Automatically generated names for subscriptions, publications, and
+       replication slots are used when this option is specified. This option
+       cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+        <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +110,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option>
+       an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +271,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +288,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +304,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index d067eb44e6c..831408872b9 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +245,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1962,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2086,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2104,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2205,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2280,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 2c9bd5bdb9e..f32d254b4e8 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,75 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# Create a new database on node_p
+$node_p->safe_psql(
+	"postgres", qq(
+	CREATE DATABASE db1;
+));
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+like(
+	$stderr,
+	qr/.*pg_createsubscriber: creating publication .* in database \"postgres\"/,
+	"expanded commands for all databases");
+like(
+	$stderr,
+	qr/.*pg_createsubscriber: creating publication .* in database \"db1\"/,
+	"expanded commands for all databases");
+
+# Drop the newly created database on node_p
+$node_p->safe_psql(
+	"postgres", qq(
+	DROP DATABASE db1;
+));
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v17-0002-Additional-test-cases.patchapplication/octet-stream; name=v17-0002-Additional-test-cases.patchDownload
From 2c8397d2c10d2067ee45751c375e87ac132fa1e7 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 17:41:48 +0530
Subject: [PATCH v17 2/2] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 132 ++++++++++++++++++
 1 file changed, 132 insertions(+)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f32d254b4e8..08992d94b71 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +436,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # Create a new database on node_p
 $node_p->safe_psql(
 	"postgres", qq(
@@ -549,10 +600,91 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# Stop $node_s
+$node_s->stop;
+
+# Drop one of the databases (e.g., $db2)
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+# Get subscription names
+$result = $node_u->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

v17-0003-Synopsis-for-all-option.patchapplication/octet-stream; name=v17-0003-Synopsis-for-all-option.patchDownload
From 287f4d3497782ac0262c01f34a180f5afc16b2d8 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v17 3/3] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 3220b0cfa7b..3e579ddfd2e 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

#80Shubham Khanna
khannashubham1197@gmail.com
In reply to: Amit Kapila (#78)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 5:54 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Thu, Mar 20, 2025 at 4:53 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 10:25 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--database cannot be used with --all/,
+ 'fail if --database is used with --all');

Why does only this test not use --dry-run, but all other such tests
use --dry-run? Either they should all use it or not use it.

I think this patch is adding a lot of extra tests which I don't think
are required. We should have one positive test in --dry-run mode
similar to ('run pg_createsubscriber without --databases') and
probably verify in the LOG that it has expanded commands for all
databases. Also, for negative tests, one or two tests are sufficient
instead of testing all possible combinations. I don't expect this new
option to add a noticeable testing time.

As per your suggestions, I have updated the test cases in the v17-0001
patch. The primary tests now focus on a positive test in --dry-run
mode, similar to the one for run pg_createsubscriber without
--databases, along with verification in the log to ensure that
commands for all databases are expanded. Also, I have limited the
negative tests to just one or two representative cases, avoiding
unnecessary combinations.
I have moved the additional test cases to the v17-0002 patch to ensure
that the testing time for the new option remains negligible.

*  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>

Most of this is unrelated to this patch. I suggest making a top-up
patch, we can commit it separately.

I have created a separate v17-0003 patch that includes the synopsis
for the '--all' option.

The attached patch at [1]/messages/by-id/CAHv8RjLj0KxVHbxaPZHzudGS1igzDMccFE8LDh4LHNJR_2Aqug@mail.gmail.com contains the suggested changes.

[1]: /messages/by-id/CAHv8RjLj0KxVHbxaPZHzudGS1igzDMccFE8LDh4LHNJR_2Aqug@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#81Shubham Khanna
khannashubham1197@gmail.com
In reply to: vignesh C (#75)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 3:41 PM vignesh C <vignesh21@gmail.com> wrote:

On Thu, 20 Mar 2025 at 10:25, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

I have created two patches, v16-0001 and v16-0002, to address the
performance issue. I conducted performance testing, and here are the
results:
- The difference in execution time between HEAD and the v15 patch was 53.2%.
- After removing the suggested test cases, the difference reduced to
36.43%, showing a significant improvement.

It is still taking quite a while, can we compare with the following
too see how much extra time it takes:
a) remove insert and select verification for the all database
successful tests as all of these are logical replication verification
which is extensively tested b) remove command_fails_like failure tests
c) remove both of above and see.

As suggested by Amit at [1]/messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com, I have reduced the number of test cases
in the v17-0001 patch to ensure they are concise and focused.
I conducted performance testing on the current HEAD and the latest
v17-0001 patch (available at [2]/messages/by-id/CAHv8RjLj0KxVHbxaPZHzudGS1igzDMccFE8LDh4LHNJR_2Aqug@mail.gmail.com). Here are the observations based on
five runs:-
- The difference in execution time between HEAD and the v17-0001 patch
was 1.62 seconds.
- This corresponds to a 22.44% difference in performance.

[1]: /messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com
[2]: /messages/by-id/CAHv8RjLj0KxVHbxaPZHzudGS1igzDMccFE8LDh4LHNJR_2Aqug@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#82vignesh C
vignesh21@gmail.com
In reply to: Shubham Khanna (#79)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, 21 Mar 2025 at 18:59, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

During the recent testing, I observed that the tests were failing when
using wait_for_slot_catchup(). To address this, I reverted to using
wait_for_subscription_sync(), which was employed previously and has
proven to be more reliable in ensuring test stability.
Please let me know if there are any additional adjustments you would
suggest or if you would like me to investigate further into
wait_for_slot_catchup().

I have created a separate patch for the synopsis of '--all' option as
suggested by Amit at [1]. The attached patch contains the suggested
changes.

I believe you added the following because pattern matching is
difficult for the $db1 and $db2 variables having special names:
+# Create a new database on node_p
+$node_p->safe_psql(
+       "postgres", qq(
+       CREATE DATABASE db1;
+));

How about we change it to verify the count of occurrences instead for
this case like below:
# Verify that the required logical replication objects are created. The
# expected count 3 refers to postgres, $db1 and $db2 databases.
is(scalar(() = $stderr =~ /creating publication/g),
3, "verify publications are created for all databases");
is(scalar(() = $stderr =~ /creating the replication slot/g),
3, "verify replication slots are created for all databases");
is(scalar(() = $stderr =~ /creating subscription/g),
3, "verify subscriptions are created for all databases");

If you are ok, you can merge the changes attached.

Regards,
Vignesh

Attachments:

vignesh_review_comment_fixes.patchtext/x-patch; charset=US-ASCII; name=vignesh_review_comment_fixes.patchDownload
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 7e275473eb9..9355cb0ade9 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -419,12 +419,6 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
-# Create a new database on node_p
-$node_p->safe_psql(
-	"postgres", qq(
-	CREATE DATABASE db1;
-));
-
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -440,20 +434,14 @@ my ($stdout, $stderr) = run_command(
 	],
 	'run pg_createsubscriber with --all');
 
-like(
-	$stderr,
-	qr/.*pg_createsubscriber: creating publication .* in database \"postgres\"/,
-	"expanded commands for all databases");
-like(
-	$stderr,
-	qr/.*pg_createsubscriber: creating publication .* in database \"db1\"/,
-	"expanded commands for all databases");
-
-# Drop the newly created database on node_p
-$node_p->safe_psql(
-	"postgres", qq(
-	DROP DATABASE db1;
-));
+# Verify that the required logical replication objects are created. The
+# expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
 
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
#83Shubham Khanna
khannashubham1197@gmail.com
In reply to: vignesh C (#82)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Sat, Mar 22, 2025 at 6:23 PM vignesh C <vignesh21@gmail.com> wrote:

On Fri, 21 Mar 2025 at 18:59, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

During the recent testing, I observed that the tests were failing when
using wait_for_slot_catchup(). To address this, I reverted to using
wait_for_subscription_sync(), which was employed previously and has
proven to be more reliable in ensuring test stability.
Please let me know if there are any additional adjustments you would
suggest or if you would like me to investigate further into
wait_for_slot_catchup().

I have created a separate patch for the synopsis of '--all' option as
suggested by Amit at [1]. The attached patch contains the suggested
changes.

I believe you added the following because pattern matching is
difficult for the $db1 and $db2 variables having special names:
+# Create a new database on node_p
+$node_p->safe_psql(
+       "postgres", qq(
+       CREATE DATABASE db1;
+));

How about we change it to verify the count of occurrences instead for
this case like below:
# Verify that the required logical replication objects are created. The
# expected count 3 refers to postgres, $db1 and $db2 databases.
is(scalar(() = $stderr =~ /creating publication/g),
3, "verify publications are created for all databases");
is(scalar(() = $stderr =~ /creating the replication slot/g),
3, "verify replication slots are created for all databases");
is(scalar(() = $stderr =~ /creating subscription/g),
3, "verify subscriptions are created for all databases");

If you are ok, you can merge the changes attached.

I agree that verifying the count of occurrences is a more
straightforward and effective approach, especially given the
challenges with pattern matching for $db1 and $db2 variables with
special names. This method simplifies validation and enhances
robustness by explicitly ensuring the expected number of logical
replication objects are created.

I have reviewed and merged the proposed changes into the patch. The
attached patches contain the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v18-0003-Synopsis-for-all-option.patchapplication/octet-stream; name=v18-0003-Synopsis-for-all-option.patchDownload
From 9b6d7bdd0e3e0a8ff7e0b4faa74d30680d824a39 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v18 3/3] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index aac2b66b1a9..e6ef7e50046 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

v18-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v18-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From 5b50f4694fa5f97bd3d3f6cf2c7b633215c0dd92 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v18] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     | 35 +++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 92 ++++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              | 57 ++++++++++++
 3 files changed, 173 insertions(+), 11 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d011b79e5e6..aac2b66b1a9 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
+       Automatically generated names for subscriptions, publications, and
+       replication slots are used when this option is specified. This option
+       cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+        <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +110,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option>
+       an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +271,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +288,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +304,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index e2d6b7544bf..6dbe0b87078 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +245,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1962,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2086,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2104,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2205,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2280,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f7a980ec799..9355cb0ade9 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,63 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Verify that the required logical replication objects are created. The
+# expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v18-0002-Additional-test-cases.patchapplication/octet-stream; name=v18-0002-Additional-test-cases.patchDownload
From 8a82f0b7cd9d6e88c92ade7d414a651532c3ef3b Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Sat, 22 Mar 2025 19:08:30 +0530
Subject: [PATCH v18 2/2] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 132 ++++++++++++++++++
 1 file changed, 132 insertions(+)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 9355cb0ade9..85b36487045 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +436,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -537,10 +588,91 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# Stop $node_s
+$node_s->stop;
+
+# Drop one of the databases (e.g., $db2)
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+# Get subscription names
+$result = $node_u->safe_psql(
+	'postgres', qq(
+	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

#84Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#83)
1 attachment(s)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

I have reviewed and merged the proposed changes into the patch. The
attached patches contain the suggested changes.

Thanks for updating the patch! Few comments:

01.
```
+       /*
+        * Fetch all databases from the source (publisher) if --all is specified.
+        */
+       if (opt.all_dbs)
+               fetch_source_databases(&opt);
+
        if (opt.database_names.head == NULL)
```

I feel we can convert "if" -> "else if" for the opt.database_names.head case,
because fetch_source_databases() ensures databases are listed.

02.
```
+# Set up node U as standby linking to node
```

To confirm: why can we use "U" here?

03.
```
+$node_u->append_conf(
+       'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
```
I think this part is not required. init_from_backup() with has_streaming sets the
primary_conninfo.

04.
```
+# Stop $node_s
+$node_s->stop;
```

The comment does not describe anything. I feel you can say "Stop node S because
it won't be used anymore", or just remove it.

05.
```
+# Drop one of the databases (e.g., $db2)
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
```

"e.g." means "for example", so it is not suitable to put here. Please explain why it must be removed.

06.
```
+# Get subscription names
+$result = $node_u->safe_psql(
+       'postgres', qq(
+       SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
```

wait_for_subscription_sync() is used to wait until the initial synchronization is
done, so not suitable here. wait_for_catchup() is more appropriate.

07.
Also, the similar part exists on the pre-existing part (wait_for_subscription_sync
was used there, but I feel this may not be correct). How about unifing them like
attached?

08.
```
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
```

This is confusing because 'fourth row' is inserted to $db1 just after it. How
about the 'row in database postgres' or something?

Best regards,
Hayato Kuroda
FUJITSU LIMITED

Attachments:

kuroda.diffsapplication/octet-stream; name=kuroda.diffsDownload
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 85b36487045..e1f9c2f9543 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -41,6 +41,25 @@ sub generate_db
 	return $dbname;
 }
 
+# Wait for subscriptions on the subscriber to catch up all changes.
+sub wait_for_all_subscriptions_caught_up
+{
+	my ($node_p, $node_s) = @_;
+
+	# Get subscription names
+	my $result = $node_s->safe_psql(
+		'postgres', qq(
+		SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+	));
+	my @subnames = split("\n", $result);
+
+	# Wait for all subscriptions to catch up
+	foreach my $subname (@subnames)
+	{
+		$node_p->wait_for_catchup($subname);
+	}
+}
+
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -553,16 +572,7 @@ $result = $node_s->safe_psql(
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_s);
 
 # Confirm the failover slot has been removed
 $result = $node_s->safe_psql($db1,
@@ -601,10 +611,6 @@ $node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
 $node_p->backup('backup_3');
 my $node_u = PostgreSQL::Test::Cluster->new('node_u');
 $node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
-$node_u->append_conf(
-	'postgresql.conf', qq[
-primary_conninfo = '$pconnstr dbname=postgres'
-]);
 $node_u->set_standby_mode();
 
 # run pg_createsubscriber with '--all' option without '--dry-run'
@@ -645,16 +651,7 @@ is($result, '0', 'subscription is not created on template databases');
 $node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
 $node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
 
-# Get subscription names
-$result = $node_u->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames1 = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
-$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_u);
 
 # Check result in database 'postgres' of node U
 $result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
#85Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#84)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 24, 2025 at 12:29 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

I have reviewed and merged the proposed changes into the patch. The
attached patches contain the suggested changes.

Thanks for updating the patch! Few comments:

01.
```
+       /*
+        * Fetch all databases from the source (publisher) if --all is specified.
+        */
+       if (opt.all_dbs)
+               fetch_source_databases(&opt);
+
if (opt.database_names.head == NULL)
```

I feel we can convert "if" -> "else if" for the opt.database_names.head case,
because fetch_source_databases() ensures databases are listed.

Fixed.

02.
```
+# Set up node U as standby linking to node
```

To confirm: why can we use "U" here?

Since node_s and node_t are already used in this test, I have used the
next letter, node_u. Additionally, comments have been added to improve
clarity.

03.
```
+$node_u->append_conf(
+       'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
```
I think this part is not required. init_from_backup() with has_streaming sets the
primary_conninfo.

Fixed.

04.
```
+# Stop $node_s
+$node_s->stop;
```

The comment does not describe anything. I feel you can say "Stop node S because
it won't be used anymore", or just remove it.

Fixed.

05.
```
+# Drop one of the databases (e.g., $db2)
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
```

"e.g." means "for example", so it is not suitable to put here. Please explain why it must be removed.

Fixed.

06.
```
+# Get subscription names
+$result = $node_u->safe_psql(
+       'postgres', qq(
+       SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
```

wait_for_subscription_sync() is used to wait until the initial synchronization is
done, so not suitable here. wait_for_catchup() is more appropriate.

Fixed.

07.
Also, the similar part exists on the pre-existing part (wait_for_subscription_sync
was used there, but I feel this may not be correct). How about unifing them like
attached?

Fixed.

08.
```
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
```

This is confusing because 'fourth row' is inserted to $db1 just after it. How
about the 'row in database postgres' or something?

Fixed.

The attached patches contain the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v19-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v19-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From 89b481eec9e9d04a72c8fec15cd2fbfee4e72d99 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v19 1/3] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     | 35 +++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 94 ++++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              | 57 +++++++++++
 3 files changed, 174 insertions(+), 12 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d011b79e5e6..aac2b66b1a9 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
+       Automatically generated names for subscriptions, publications, and
+       replication slots are used when this option is specified. This option
+       cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+        <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +110,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option>
+       an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +271,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +288,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +304,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index e2d6b7544bf..41a95f8cc01 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +245,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1962,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2086,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2104,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2205,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2280,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
-	if (opt.database_names.head == NULL)
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
+	else if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f7a980ec799..9355cb0ade9 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,63 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Verify that the required logical replication objects are created. The
+# expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v19-0002-Synopsis-for-all-option.patchapplication/octet-stream; name=v19-0002-Synopsis-for-all-option.patchDownload
From 40e8904f39372d4fcbae17afd76ca21da876c5fc Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v19 2/3] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index aac2b66b1a9..e6ef7e50046 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

v19-0003-Additional-test-cases.patchapplication/octet-stream; name=v19-0003-Additional-test-cases.patchDownload
From 77f0b90263756fdce856e0e677d12de609f36525 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Sat, 22 Mar 2025 19:08:30 +0530
Subject: [PATCH v19 3/3] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 151 ++++++++++++++++--
 1 file changed, 141 insertions(+), 10 deletions(-)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 9355cb0ade9..72a0318d71e 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -41,6 +41,25 @@ sub generate_db
 	return $dbname;
 }
 
+# Wait for subscriptions on the subscriber to catch up all changes.
+sub wait_for_all_subscriptions_caught_up
+{
+	my ($node_p, $node_s) = @_;
+
+	# Get subscription names
+	my $result = $node_s->safe_psql(
+		'postgres', qq(
+		SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+	));
+	my @subnames = split("\n", $result);
+
+	# Wait for all subscriptions to catch up
+	foreach my $subname (@subnames)
+	{
+		$node_p->wait_for_catchup($subname);
+	}
+}
+
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -386,6 +405,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +455,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -502,16 +572,7 @@ $result = $node_s->safe_psql(
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_s);
 
 # Confirm the failover slot has been removed
 $result = $node_s->safe_psql($db1,
@@ -537,10 +598,80 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+$node_s->stop;
+
+# Drop the database $db2 to verify subscriptions are handled correctly
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create a test table
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres',
+	"INSERT INTO tbl1 VALUES('row in database postgres')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+wait_for_all_subscriptions_caught_up($node_p, $node_u);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is( $result,
+	qq(row in database postgres),
+	"logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.34.1

#86Shlok Kyal
shlok.kyal.oss@gmail.com
In reply to: Shubham Khanna (#85)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, 24 Mar 2025 at 15:56, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Mon, Mar 24, 2025 at 12:29 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

I have reviewed and merged the proposed changes into the patch. The
attached patches contain the suggested changes.

Thanks for updating the patch! Few comments:

01.
```
+       /*
+        * Fetch all databases from the source (publisher) if --all is specified.
+        */
+       if (opt.all_dbs)
+               fetch_source_databases(&opt);
+
if (opt.database_names.head == NULL)
```

I feel we can convert "if" -> "else if" for the opt.database_names.head case,
because fetch_source_databases() ensures databases are listed.

Fixed.

02.
```
+# Set up node U as standby linking to node
```

To confirm: why can we use "U" here?

Since node_s and node_t are already used in this test, I have used the
next letter, node_u. Additionally, comments have been added to improve
clarity.

03.
```
+$node_u->append_conf(
+       'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
```
I think this part is not required. init_from_backup() with has_streaming sets the
primary_conninfo.

Fixed.

04.
```
+# Stop $node_s
+$node_s->stop;
```

The comment does not describe anything. I feel you can say "Stop node S because
it won't be used anymore", or just remove it.

Fixed.

05.
```
+# Drop one of the databases (e.g., $db2)
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
```

"e.g." means "for example", so it is not suitable to put here. Please explain why it must be removed.

Fixed.

06.
```
+# Get subscription names
+$result = $node_u->safe_psql(
+       'postgres', qq(
+       SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
```

wait_for_subscription_sync() is used to wait until the initial synchronization is
done, so not suitable here. wait_for_catchup() is more appropriate.

Fixed.

07.
Also, the similar part exists on the pre-existing part (wait_for_subscription_sync
was used there, but I feel this may not be correct). How about unifing them like
attached?

Fixed.

08.
```
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
```

This is confusing because 'fourth row' is inserted to $db1 just after it. How
about the 'row in database postgres' or something?

Fixed.

The attached patches contain the suggested changes.

I have reviewed the v19-0001 patch I have a comment:

In pg_createsubscriber.sgml, this line seems little odd:

+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option>
+       an error will be reported.

Should we change the sentence to something like:

+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.

Thanks,
Shlok Kyal

#87Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#79)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Fri, Mar 21, 2025 at 6:59 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Thu, Mar 20, 2025 at 4:53 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 10:25 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.

I have started reviewing the patches again. Here are some review comments

<variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.

In this construct "all" goes with "source server" but it should go
with "non-template database". How about, "For every non-template
database on the source server, create one subscription on the target
server in the database with the same name."?

Fixed.

+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

... used along with ...

also comma before or .

We should also mention that generated names of replication slots,
publications and subscriptions are used when using this option.

+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-d <replaceable
class="parameter">dbname</replaceable></option></term>
<term><option>--database=<replaceable
class="parameter">dbname</replaceable></option></term>

Fixed.

@@ -94,9 +130,10 @@ PostgreSQL documentation
<para>
The name of the database in which to create a subscription.  Multiple
databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with
<option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
<option>-P</option> option, an error will be reported.

We have to mention -a as well here. Something like "If the database
name is not specified in either the <option>-d</option> option or
<option>-P</option> option, and <option>-a</option> is not provided,
an error will be reported."

Fixed.

+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--database cannot be used with --all/,
+ 'fail if --database is used with --all');

Why does only this test not use --dry-run, but all other such tests
use --dry-run? Either they should all use it or not use it.

+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--publication' => 'pub1',
+ ],
+ qr/--publication cannot be used with --all/,
+ 'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--replication-slot' => 'replslot1',
+ '--all',
+ ],
+ qr/--replication-slot cannot be used with --all/,
+ 'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--subscription' => 'sub1',
+ ],
+ qr/--subscription cannot be used with --all/,
+ 'fail if --subscription is used with --all');
+

Just to cover all combinations we need to test both scenarios. where
--all comes before an incompatible option and vice versa.

# Run pg_createsubscriber on node S. --verbose is used twice
# to show more information.
# In passing, also test the --enable-two-phase option
@@ -459,10 +526,93 @@ my $sysid_s = $node_s->safe_psql('postgres',
'SELECT system_identifier FROM pg_control_system()');
ok($sysid_p != $sysid_s, 'system identifier was changed');

+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');

$db1 and $db2 already have tables. We can avoid creating new ones here.

Fixed.

+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+

This function just makes sure that initial sync is done. But it
doesn't wait for replication to catch up with current state of the
publisher. It's the latter which would make sure that the last INSERTS
are visible. We should be using wait_for_slot_catchup() on the
publisher.

+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+ "logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+

These tests may not always pass if we use wait_for_slot_catchup above.

During the recent testing, I observed that the tests were failing when
using wait_for_slot_catchup(). To address this, I reverted to using
wait_for_subscription_sync(), which was employed previously and has
proven to be more reliable in ensuring test stability.
Please let me know if there are any additional adjustments you would
suggest or if you would like me to investigate further into
wait_for_slot_catchup().

wait_for_subscription_sync() calls wait_for_catchup() which actually
waits for a given WAL sender to pass given LSN. So it's working fine.
But wait_for_subscription_sync() is used to make sure that the initial
data sync is completed (at least in the cases that I looked at) not
that the regular replication has caught up. And it's doing more work
that required. I see other tests using wait_for_catchup() for this
purpose. Should we use that with appropriate arguments?

--
Best Wishes,
Ashutosh Bapat

#88Shubham Khanna
khannashubham1197@gmail.com
In reply to: Shlok Kyal (#86)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 24, 2025 at 5:41 PM Shlok Kyal <shlok.kyal.oss@gmail.com> wrote:

On Mon, 24 Mar 2025 at 15:56, Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Mon, Mar 24, 2025 at 12:29 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

I have reviewed and merged the proposed changes into the patch. The
attached patches contain the suggested changes.

Thanks for updating the patch! Few comments:

01.
```
+       /*
+        * Fetch all databases from the source (publisher) if --all is specified.
+        */
+       if (opt.all_dbs)
+               fetch_source_databases(&opt);
+
if (opt.database_names.head == NULL)
```

I feel we can convert "if" -> "else if" for the opt.database_names.head case,
because fetch_source_databases() ensures databases are listed.

Fixed.

02.
```
+# Set up node U as standby linking to node
```

To confirm: why can we use "U" here?

Since node_s and node_t are already used in this test, I have used the
next letter, node_u. Additionally, comments have been added to improve
clarity.

03.
```
+$node_u->append_conf(
+       'postgresql.conf', qq[
+primary_conninfo = '$pconnstr dbname=postgres'
+]);
```
I think this part is not required. init_from_backup() with has_streaming sets the
primary_conninfo.

Fixed.

04.
```
+# Stop $node_s
+$node_s->stop;
```

The comment does not describe anything. I feel you can say "Stop node S because
it won't be used anymore", or just remove it.

Fixed.

05.
```
+# Drop one of the databases (e.g., $db2)
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
```

"e.g." means "for example", so it is not suitable to put here. Please explain why it must be removed.

Fixed.

06.
```
+# Get subscription names
+$result = $node_u->safe_psql(
+       'postgres', qq(
+       SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+));
+my @subnames1 = split("\n", $result);
+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
```

wait_for_subscription_sync() is used to wait until the initial synchronization is
done, so not suitable here. wait_for_catchup() is more appropriate.

Fixed.

07.
Also, the similar part exists on the pre-existing part (wait_for_subscription_sync
was used there, but I feel this may not be correct). How about unifing them like
attached?

Fixed.

08.
```
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres', "INSERT INTO tbl1 VALUES('first row')");
```

This is confusing because 'fourth row' is inserted to $db1 just after it. How
about the 'row in database postgres' or something?

Fixed.

The attached patches contain the suggested changes.

I have reviewed the v19-0001 patch I have a comment:

In pg_createsubscriber.sgml, this line seems little odd:

+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option>
+       an error will be reported.

Should we change the sentence to something like:

+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.

Fixed.

The attached patches contain the suggested changes.

Thanks and regards,
Shubham Khanna.

Attachments:

v20-0003-Additional-test-cases.patchapplication/octet-stream; name=v20-0003-Additional-test-cases.patchDownload
From c4d35243ee4efb26e3433b88205a5661db3f3b44 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Sat, 22 Mar 2025 19:08:30 +0530
Subject: [PATCH v20 3/3] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 151 ++++++++++++++++--
 1 file changed, 141 insertions(+), 10 deletions(-)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index 9355cb0ade9..72a0318d71e 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -41,6 +41,25 @@ sub generate_db
 	return $dbname;
 }
 
+# Wait for subscriptions on the subscriber to catch up all changes.
+sub wait_for_all_subscriptions_caught_up
+{
+	my ($node_p, $node_s) = @_;
+
+	# Get subscription names
+	my $result = $node_s->safe_psql(
+		'postgres', qq(
+		SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+	));
+	my @subnames = split("\n", $result);
+
+	# Wait for all subscriptions to catch up
+	foreach my $subname (@subnames)
+	{
+		$node_p->wait_for_catchup($subname);
+	}
+}
+
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -386,6 +405,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +455,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -502,16 +572,7 @@ $result = $node_s->safe_psql(
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_s);
 
 # Confirm the failover slot has been removed
 $result = $node_s->safe_psql($db1,
@@ -537,10 +598,80 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+$node_s->stop;
+
+# Drop the database $db2 to verify subscriptions are handled correctly
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create a test table
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres',
+	"INSERT INTO tbl1 VALUES('row in database postgres')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+wait_for_all_subscriptions_caught_up($node_p, $node_u);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is( $result,
+	qq(row in database postgres),
+	"logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

v20-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v20-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From d14af9738d6c04dc1d5dd0d9d174944a4603512d Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v20] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     | 35 +++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 94 ++++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              | 57 +++++++++++
 3 files changed, 174 insertions(+), 12 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d011b79e5e6..5665672772d 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,22 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
+       Automatically generated names for subscriptions, publications, and
+       replication slots are used when this option is specified. This option
+       cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+        <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +110,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +271,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +288,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +304,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index e2d6b7544bf..41a95f8cc01 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +245,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1962,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2086,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2104,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2205,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2280,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
-	if (opt.database_names.head == NULL)
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
+	else if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f7a980ec799..9355cb0ade9 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,63 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Verify that the required logical replication objects are created. The
+# expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v20-0002-Synopsis-for-all-option.patchapplication/octet-stream; name=v20-0002-Synopsis-for-all-option.patchDownload
From 04163d72578f0db6a917f476fdd5c20643bb675f Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v20 2/2] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 5665672772d..e5ba1e1c777 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

#89Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#87)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Mon, Mar 24, 2025 at 6:08 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Fri, Mar 21, 2025 at 6:59 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Thu, Mar 20, 2025 at 4:53 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 10:25 AM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.

I have started reviewing the patches again. Here are some review comments

<variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       For all source server non-template databases create subscriptions for
+       databases with the same names on the target server.

In this construct "all" goes with "source server" but it should go
with "non-template database". How about, "For every non-template
database on the source server, create one subscription on the target
server in the database with the same name."?

Fixed.

+       Subscription names, publication names, and replication slot names are
+       automatically generated. This option cannot be used together with
+       <option>--database</option>, <option>--publication</option>,
+       <option>--replication-slot</option> or <option>--subscription</option>.

... used along with ...

also comma before or .

We should also mention that generated names of replication slots,
publications and subscriptions are used when using this option.

+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-d <replaceable
class="parameter">dbname</replaceable></option></term>
<term><option>--database=<replaceable
class="parameter">dbname</replaceable></option></term>

Fixed.

@@ -94,9 +130,10 @@ PostgreSQL documentation
<para>
The name of the database in which to create a subscription.  Multiple
databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
+       switches. This option cannot be used together with
<option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option or
<option>-P</option> option, an error will be reported.

We have to mention -a as well here. Something like "If the database
name is not specified in either the <option>-d</option> option or
<option>-P</option> option, and <option>-a</option> is not provided,
an error will be reported."

Fixed.

+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--database' => $db1,
+ '--all',
+ ],
+ qr/--database cannot be used with --all/,
+ 'fail if --database is used with --all');

Why does only this test not use --dry-run, but all other such tests
use --dry-run? Either they should all use it or not use it.

+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--publication' => 'pub1',
+ ],
+ qr/--publication cannot be used with --all/,
+ 'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--replication-slot' => 'replslot1',
+ '--all',
+ ],
+ qr/--replication-slot cannot be used with --all/,
+ 'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+ [
+ 'pg_createsubscriber',
+ '--verbose',
+ '--dry-run',
+ '--pgdata' => $node_s->data_dir,
+ '--publisher-server' => $node_p->connstr($db1),
+ '--socketdir' => $node_s->host,
+ '--subscriber-port' => $node_s->port,
+ '--all',
+ '--subscription' => 'sub1',
+ ],
+ qr/--subscription cannot be used with --all/,
+ 'fail if --subscription is used with --all');
+

Just to cover all combinations we need to test both scenarios. where
--all comes before an incompatible option and vice versa.

# Run pg_createsubscriber on node S. --verbose is used twice
# to show more information.
# In passing, also test the --enable-two-phase option
@@ -459,10 +526,93 @@ my $sysid_s = $node_s->safe_psql('postgres',
'SELECT system_identifier FROM pg_control_system()');
ok($sysid_p != $sysid_s, 'system identifier was changed');

+# On node P create test tables
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+$node_p->safe_psql($db1, 'CREATE TABLE tbl2 (a text)');
+$node_p->safe_psql($db1, "INSERT INTO tbl2 VALUES('first row')");
+$node_p->safe_psql($db2, 'CREATE TABLE tbl3 (a text)');

$db1 and $db2 already have tables. We can avoid creating new ones here.

Fixed.

+
+# Wait subscriber to catch up
+$node_u->wait_for_subscription_sync($node_p, $subnames1[0]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[1]);
+$node_u->wait_for_subscription_sync($node_p, $subnames1[2]);
+

This function just makes sure that initial sync is done. But it
doesn't wait for replication to catch up with current state of the
publisher. It's the latter which would make sure that the last INSERTS
are visible. We should be using wait_for_slot_catchup() on the
publisher.

+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is($result, qq(first row), "logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl2');
+is( $result, qq(first row
+second row),
+ "logical replication works in database $db1");
+
+# Check result in database $db2 of node U
+$result = $node_u->safe_psql($db2, 'SELECT * FROM tbl3');
+is($result, qq(first row), "logical replication works in database $db2");
+

These tests may not always pass if we use wait_for_slot_catchup above.

During the recent testing, I observed that the tests were failing when
using wait_for_slot_catchup(). To address this, I reverted to using
wait_for_subscription_sync(), which was employed previously and has
proven to be more reliable in ensuring test stability.
Please let me know if there are any additional adjustments you would
suggest or if you would like me to investigate further into
wait_for_slot_catchup().

wait_for_subscription_sync() calls wait_for_catchup() which actually
waits for a given WAL sender to pass given LSN. So it's working fine.
But wait_for_subscription_sync() is used to make sure that the initial
data sync is completed (at least in the cases that I looked at) not
that the regular replication has caught up. And it's doing more work
that required. I see other tests using wait_for_catchup() for this
purpose. Should we use that with appropriate arguments?

--

The v20-0003 patch attached at [1]/messages/by-id/CAHv8RjJ4AQCYRanC-9rX9xjZnzH0LWgFyS6Ve-1-gHNG-i5=AQ@mail.gmail.com contains the suggested changes.

[1]: /messages/by-id/CAHv8RjJ4AQCYRanC-9rX9xjZnzH0LWgFyS6Ve-1-gHNG-i5=AQ@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#90Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#88)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

The attached patches contain the suggested changes.

Thanks for updating the patch. I reviewed only 0001 because they would be committed separately.
Few comments:

01.
```
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
```

It is quite confusing for me; We do not have to describe users that this command
checks databases of the source server. How about something like:
"Create one subscription per all non-template databases on the target server."

02.
```
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
```

You seemed to move most of validation checks to 0002. Can you tell me a reason
why they are remained?

03.
```
+# Verify that the required logical replication objects are created. The
+# expected count 3 refers to postgres, $db1 and $db2 databases.
```

Hmm, but since we did a dry-run, any objects are not created, right?

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#91Shubham Khanna
khannashubham1197@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#90)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 3:22 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

The attached patches contain the suggested changes.

Thanks for updating the patch. I reviewed only 0001 because they would be committed separately.
Few comments:

01.
```
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
```

It is quite confusing for me; We do not have to describe users that this command
checks databases of the source server. How about something like:
"Create one subscription per all non-template databases on the target server."

Fixed.

02.
```
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--database' => $db1,
+               '--all',
+       ],
+       qr/--database cannot be used with --all/,
+       'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--dry-run',
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--all',
+               '--publication' => 'pub1',
+       ],
+       qr/--publication cannot be used with --all/,
+       'fail if --publication is used with --all');
```

You seemed to move most of validation checks to 0002. Can you tell me a reason
why they are remained?

As per Amit's suggestions at [1]/messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com, I moved most of the validation test
cases to the separate 0003 patch. However, these particular checks
were retained in the main patch because they ensure the fundamental
validations for incompatible options (--database and --publication
with --all) are covered early on.

03.
```
+# Verify that the required logical replication objects are created. The
+# expected count 3 refers to postgres, $db1 and $db2 databases.
```

Hmm, but since we did a dry-run, any objects are not created, right?

Fixed.

The attached patches contain the suggested changes.
[1]: /messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com

Thanks and regards,
Shubham Khanna.

#92Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#91)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 4:28 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.
[1] - /messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com

Forgot to attach patches?

--
Best Wishes,
Ashutosh Bapat

#93Shubham Khanna
khannashubham1197@gmail.com
In reply to: Ashutosh Bapat (#92)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 4:31 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Tue, Mar 25, 2025 at 4:28 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.
[1] - /messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com

Forgot to attach patches?

--

Apologies for the oversight. I have attached the patches now. Please
find them included here.

Thanks and regards,
Shubham Khanna.

Attachments:

v21-0002-Synopsis-for-all-option.patchapplication/octet-stream; name=v21-0002-Synopsis-for-all-option.patchDownload
From 10de52bf1f764551dd834e05053b14f3f85bc8c1 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v21 2/2] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 5ea399cee62..c2a3614f056 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

v21-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v21-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From 43ec40ee2af8b0e5562129a5bdf53ca123a7bcc3 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v21] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     | 34 +++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 94 ++++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              | 57 +++++++++++
 3 files changed, 173 insertions(+), 12 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d011b79e5e6..5ea399cee62 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,21 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       Create one subscription per all non-template databases on the target
+       server. Automatically generated names for subscriptions, publications,
+       and replication slots are used when this option is specified. This
+       option cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+       <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +109,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +270,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +287,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +303,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index e2d6b7544bf..41a95f8cc01 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* --all option was specified */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_source_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +245,7 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all non-template source databases\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1962,60 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connect_database(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	/* Process the query result */
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	/* Error if no databases were found on the source server */
+	if (num_dbs == 0)
+	{
+		pg_log_error("no suitable databases found on the source server");
+		pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2086,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2104,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2205,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2280,20 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
-	if (opt.database_names.head == NULL)
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 */
+	if (opt.all_dbs)
+		fetch_source_databases(&opt);
+
+	else if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f7a980ec799..a00a24eb4c1 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,63 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Verify that the required logical replication objects are output.
+# The expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v21-0003-Additional-test-cases.patchapplication/octet-stream; name=v21-0003-Additional-test-cases.patchDownload
From f794e332ec5d5390604215d111a5a11b042f3c31 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Sat, 22 Mar 2025 19:08:30 +0530
Subject: [PATCH v21 3/3] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 151 ++++++++++++++++--
 1 file changed, 141 insertions(+), 10 deletions(-)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index a00a24eb4c1..a814a8b44d4 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -41,6 +41,25 @@ sub generate_db
 	return $dbname;
 }
 
+# Wait for subscriptions on the subscriber to catch up all changes.
+sub wait_for_all_subscriptions_caught_up
+{
+	my ($node_p, $node_s) = @_;
+
+	# Get subscription names
+	my $result = $node_s->safe_psql(
+		'postgres', qq(
+		SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+	));
+	my @subnames = split("\n", $result);
+
+	# Wait for all subscriptions to catch up
+	foreach my $subname (@subnames)
+	{
+		$node_p->wait_for_catchup($subname);
+	}
+}
+
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -386,6 +405,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +455,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -502,16 +572,7 @@ $result = $node_s->safe_psql(
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_s);
 
 # Confirm the failover slot has been removed
 $result = $node_s->safe_psql($db1,
@@ -537,10 +598,80 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+$node_s->stop;
+
+# Drop the database $db2 to verify subscriptions are handled correctly
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create a test table
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres',
+	"INSERT INTO tbl1 VALUES('row in database postgres')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+wait_for_all_subscriptions_caught_up($node_p, $node_u);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is( $result,
+	qq(row in database postgres),
+	"logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

#94Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Hayato Kuroda (Fujitsu) (#90)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 3:22 PM Hayato Kuroda (Fujitsu)
<kuroda.hayato@fujitsu.com> wrote:

Dear Shubham,

The attached patches contain the suggested changes.

Thanks for updating the patch. I reviewed only 0001 because they would be committed separately.
Few comments:

01.
```
+       For every non-template database on the source server, create one
+       subscription on the target server in the database with the same name.
```

It is quite confusing for me; We do not have to describe users that this command
checks databases of the source server. How about something like:
"Create one subscription per all non-template databases on the target server."

The new description doesn't mention the link between the source and
target database. And I think it's incorrect. Not all databases on the
target server will receive a subscription. Only those which have the
same name as a database on the source server. Am I correct? I prefer
the previous wording, even if it's a bit complex, over a simpler but
incorrect description.

--
Best Wishes,
Ashutosh Bapat

#95Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Shubham Khanna (#93)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.
Comments on 0001
@@ -87,6 +87,21 @@ PostgreSQL documentation
command-line arguments:
<variablelist>
+ <varlistentry>
+ <term><option>-a</option></term>
+ <term><option>--all</option></term>
+ <listitem>
+ <para>
+ Create one subscription per all non-template databases on the target
+ server. Automatically generated names for subscriptions, publications,

Already provided comment on this part.

+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server.
+ */

The prologue of this function shouldn't mention --all and --database
option as it doesn't deal with those option itself. I would just say
"Fetch a list of all not-template databases from the source server.

/* Any non-option arguments? */
if (optind < argc)
{
@@ -2202,14 +2280,20 @@ main(int argc, char **argv)
pg_log_info("validating subscriber connection string");
sub_base_conninfo = get_sub_conninfo(&opt);
- if (opt.database_names.head == NULL)
+ /*
+ * Fetch all databases from the source (publisher) if --all is specified.

Add the relevant parts from fetch_source_databases() prologue here.
That is, just add, "This is treated as if the user specified multiple
--database options, one for each source database.".

+ */
+ if (opt.all_dbs)
+ fetch_source_databases(&opt);
+

extra line, not needed

This looks mostly ready except the test changes. I believe when
committing, we are going to squash all three into a single commit. Is
that correct?

On Tue, Mar 25, 2025 at 4:37 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

On Tue, Mar 25, 2025 at 4:31 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Tue, Mar 25, 2025 at 4:28 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes.
[1] - /messages/by-id/CAA4eK1KUDEO0t6i16_CcEpg33sgcgEddHcdVC_q8j4tVUb5FWw@mail.gmail.com

Forgot to attach patches?

--

Apologies for the oversight. I have attached the patches now. Please
find them included here.

Thanks and regards,
Shubham Khanna.

--
Best Wishes,
Ashutosh Bapat

#96Amit Kapila
amit.kapila16@gmail.com
In reply to: Ashutosh Bapat (#95)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 5:08 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

This looks mostly ready except the test changes. I believe when
committing, we are going to squash all three into a single commit. Is
that correct?

I would not prefer to commit 0003 as it is primarily because of test
+# run pg_createsubscriber with '--all' option without '--dry-run'. I
am not so sure whether it is really worth adding test cycles for this
option except for dry-run mode tests but we can discuss after
committing the core patch.

--
With Regards,
Amit Kapila.

#97Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Amit Kapila (#96)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 5:15 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Tue, Mar 25, 2025 at 5:08 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

This looks mostly ready except the test changes. I believe when
committing, we are going to squash all three into a single commit. Is
that correct?

I would not prefer to commit 0003 as it is primarily because of test
+# run pg_createsubscriber with '--all' option without '--dry-run'. I
am not so sure whether it is really worth adding test cycles for this
option except for dry-run mode tests but we can discuss after
committing the core patch.

I am worried that without that test, we won't catch bugs in creating
slots, publications and subscriptions and thus causing problems in a
setup with --all. But as long as we address that concern I am ok to
defer it after committing the core patch.

--
Best Wishes,
Ashutosh Bapat

#98Ashutosh Bapat
ashutosh.bapat.oss@gmail.com
In reply to: Amit Kapila (#78)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 20, 2025 at 5:54 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

*  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>

Most of this is unrelated to this patch. I suggest making a top-up
patch, we can commit it separately.

Isn't this documenting -a/--all option in synopsis. Why do you think
it's unrelated?

--
Best Wishes,
Ashutosh Bapat

#99Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Ashutosh Bapat (#94)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Ashutosh,

The new description doesn't mention the link between the source and
target database.

Yes, I intentionally removed.

And I think it's incorrect. Not all databases on the
target server will receive a subscription. Only those which have the
same name as a database on the source server. Am I correct?

I assumed the point that the target is the streaming standby and all changes
between them would be eventually resolved. IIUC any differences between instances
would be resolved or the command would raise an ERROR. Thus, users do not have to
consider the differences of nodes so that we can simplify the description.

Analysis
=======
If some databases are missing on the source, it would not be listed as target.
pg_createsubscriber can wait until all changes are replicated before the promotion
so these databases would be removed even on target. OK, it could work.
If some databases exist only on the source, it would be listed. This command may
fail if database written in dbinfo[0].subconninfo does not exist on the target.
Otherwise they would be eventually replicated. Looks OK.

I prefer
the previous wording, even if it's a bit complex, over a simpler but
incorrect description.

Definitely, and I know description in v20 is correct. Let's keep current
one if my idea above is wrong.

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#100Euler Taveira
euler@eulerto.com
In reply to: Shubham Khanna (#93)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025, at 8:07 AM, Shubham Khanna wrote:

Apologies for the oversight. I have attached the patches now. Please
find them included here.

I started looking at this patch. When I started reading the commit message, I
didn't understand why the options that provide names to objects are
incompatible with --all option. I agree that --all and --database should be
incompatible but the others shouldn't. We already have a check saying that the
number of specified objects cannot be different from the number of databases.
Why don't rely on it instead of forbidding these options?

/* Number of object names must match number of databases */
if (num_pubs > 0 && num_pubs != num_dbs)
{
pg_log_error("wrong number of publication names specified");
pg_log_error_detail("The number of specified publication names (%d) must match the number of specified database names (%d).",
num_pubs, num_dbs);
exit(1);
}

The following documentation is inaccurate since it doesn't say you should be
allowed to connect to the database too.

+      <para>
+       Create one subscription per all non-template databases on the target
+       server. Automatically generated names for subscriptions, publications,

My suggestion is:

Create one subscription per database on the target server. Exceptions are
template databases and databases that don't allow connections.

You are mixing short (-a) and long option (--all) on the same paragraph. It
might confuse the reader.

+       switches. This option cannot be used together with <option>--all</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.

Since there is only short options, you should also use it.

+ bool all_dbs; /* --all option was specified */
SimpleStringList objecttypes_to_remove; /* list of object types to remove */

This comment doesn't follow the same pattern as the other members. It is using
a sentence. Maybe '--all option' but it would be different from the last added
option: enable-two-phase.

I'm already thinking about translation so it sounds better if you rephrase this
description. Even if it is not precise (but that's what it is expected since if
you cannot connect to a database, it won't be possible to setup a logical
replication for it.)

+ printf(_(" -a, --all create subscriptions for all non-template source databases\n"));

Suggestion:

create subscriptions for all databases except template databases

The following comment is not accurate. The postgres database is not created by
user and will be fetched.

+/*
+ * If --all is specified, fetch a list of all user-created databases from the
+ * source server. Internally, this is treated as if the user specified multiple
+ * --database options, one for each source database.
+ */
+static void
+fetch_source_databases(struct CreateSubscriberOptions *opt)

There is no 'source' terminology in the other function names. It uses
publisher, subscriber, primary and standby. There is also other functions using
'get' so I would use get_publisher_databases as function name.

It is just a matter of style but since the columns you are using are booleans,
it sounds natural to not specify the equality operator. You should also test
datconnlimit to avoid invalid databases. I would add ORDER BY for
predictability.

+ res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn = true");

Something like:

SELECT datname FROM pg_database WHERE datistemplate AND datallowconn AND datconnlimit <> -2 ORDER BY 1

What happens if you don't specify the dbname in -P option?

+   /* Establish a connection to the source server */
+   conn = connect_database(opt->pub_conninfo_str, true);

It defaults to user name. If you are using another superuser (such as admin)
the connection might fail if there is no database named 'admin'. vacuumdb that
has a similar --all option, uses another option (--maintenance-db) to discover
which databases should be vacuumed. I'm not suggesting to add the
--maintenance-db option. I wouldn't like to hardcode a specific database
(template1, for example) if there is no dbname in the connection string.
Instead, suggest the user to specify dbname into -P option. An error message
should be provided saying the exact reason: no dbname was specified.

I don't think both comments add anything. You already explained before the
function body.

+   /* Process the query result */
+   for (int i = 0; i < PQntuples(res); i++)
+   {
+       const char *dbname = PQgetvalue(res, i, 0);
+
+       simple_string_list_append(&opt->database_names, dbname);
+
+       /* Increment num_dbs to reflect multiple --database options */
+       num_dbs++;
+   }

I wouldn't add an error here.

+   /* Error if no databases were found on the source server */
+   if (num_dbs == 0)
+   {
+       pg_log_error("no suitable databases found on the source server");
+       pg_log_error_hint("Ensure that there are non-template and connectable databases on the source server.");
+       PQclear(res);
+       disconnect_database(conn, true);
+   }

It already has a central place to provide no-database-provided errors. Use it.

if (opt.database_names.head == NULL)
{
...
}

The following code is not accurate. If I specify --all, --database and
--subscription, it will report only --database. The user will remove it and run
again. At this time, --subscription will be report. My expectation is to have
all reports at once.

+   /* Validate that --all is not used with incompatible options */
+   if (opt.all_dbs)
+   {
+       char       *bad_switch = NULL;
+
+       if (num_dbs > 0)
+           bad_switch = "--database";
+       else if (num_pubs > 0)
+           bad_switch = "--publication";
+       else if (num_replslots > 0)
+           bad_switch = "--replication-slot";
+       else if (num_subs > 0)
+           bad_switch = "--subscription";
+
+       if (bad_switch)
+       {
+           pg_log_error("%s cannot be used with --all", bad_switch);
+           pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+           exit(1);
+       }
+   }

According to my suggestion above, There won't be 'else if' and the new function
has an independent condition flow.

-   if (opt.database_names.head == NULL)
+   /*
+    * Fetch all databases from the source (publisher) if --all is specified.
+    */
+   if (opt.all_dbs)
+       fetch_source_databases(&opt);
+
+   else if (opt.database_names.head == NULL)

I checked the applications that provide multiple synopsis using the following command.

grep '<cmdsynopsis' doc/src/sgml/ref/*.sgml | awk '{print $1}' | sort | uniq -c

There are just 3 applications that have multiple cmdsynopsis. pgbench has 2
distinct tasks (load and run) so it makes sense to have 2 synopsis. pg_ctl has
multiple tasks and also deserves multiple synopsis. The complexity introduced
into vacuumdb (per table, per schema) seems to justify multiple synopsis too.
However, the same logic doesn't apply here. IMO 0002 shouldn't be applied
because the additional option (--all) doesn't justify multiple synopsis for
syntax clarification.

I have the same opinion as Amit. I wouldn't apply 0003.

--
Euler Taveira
EDB https://www.enterprisedb.com/

#101vignesh C
vignesh21@gmail.com
In reply to: Euler Taveira (#100)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, 26 Mar 2025 at 02:05, Euler Taveira <euler@eulerto.com> wrote:

On Tue, Mar 25, 2025, at 8:07 AM, Shubham Khanna wrote:

The following code is not accurate. If I specify --all, --database and
--subscription, it will report only --database. The user will remove it and run
again. At this time, --subscription will be report. My expectation is to have
all reports at once.

I felt pg_dump reports conflicts incrementally rather than listing all
incompatible options at once as shown below:
### Specified, --data-only, --schema-only, --statistics-only,
--no-data and --clean options. together
Example1: ./pg_dump --data-only --schema-only --statistics-only
--no-data --clean -d postgres -f dump.txt
pg_dump: error: options -s/--schema-only and -a/--data-only cannot be
used together

### After removing --schema-only option, another error arises
Example2: ./pg_dump --data-only --statistics-only --no-data --clean
-d postgres -f dump.txt
pg_dump: error: options -a/--data-only and --statistics-only cannot be
used together

### After removing --no-data option, another error arises
Example3: ./pg_dump --data-only --no-data --clean -d postgres -f dump.txt
pg_dump: error: options -a/--data-only and --no-data cannot be used together

### After removing --clean option, another error arises
Example4: ./pg_dump --data-only --clean -d postgres -f dump.txt
pg_dump: error: options -c/--clean and -a/--data-only cannot be used together

I believe the current patch follows this approach and also simplifies
the code handling, making it easier to maintain.

Regards,
Vignesh

#102Amit Kapila
amit.kapila16@gmail.com
In reply to: Euler Taveira (#100)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Mar 26, 2025 at 2:05 AM Euler Taveira <euler@eulerto.com> wrote:

On Tue, Mar 25, 2025, at 8:07 AM, Shubham Khanna wrote:

Apologies for the oversight. I have attached the patches now. Please
find them included here.

I started looking at this patch. When I started reading the commit message, I
didn't understand why the options that provide names to objects are
incompatible with --all option. I agree that --all and --database should be
incompatible but the others shouldn't. We already have a check saying that the
number of specified objects cannot be different from the number of databases.
Why don't rely on it instead of forbidding these options?

We must ensure to match the order of objects specified with the
database as well (The doc says: The order of the multiple publication
name switches must match the order of database switches.). It is easy
for users to do that when explicitly specifying databases with -d
switches but not with --all case. I can see a way to extend the
current functionality to allow just fetching databases from the
publisher and displaying them to the user, then we can expect the user
to specify the names of other objects in the same order but not
otherwise.

What happens if you don't specify the dbname in -P option?

+   /* Establish a connection to the source server */
+   conn = connect_database(opt->pub_conninfo_str, true);

It defaults to user name. If you are using another superuser (such as admin)
the connection might fail if there is no database named 'admin'. vacuumdb that
has a similar --all option, uses another option (--maintenance-db) to discover
which databases should be vacuumed. I'm not suggesting to add the
--maintenance-db option. I wouldn't like to hardcode a specific database
(template1, for example) if there is no dbname in the connection string.
Instead, suggest the user to specify dbname into -P option. An error message
should be provided saying the exact reason: no dbname was specified.

But why shouldn't we use the same specification as vacuumdb, which is:
"If not specified, the postgres database will be used, or if that does
not exist, template1 will be used."?

I checked the applications that provide multiple synopsis using the following command.

grep '<cmdsynopsis' doc/src/sgml/ref/*.sgml | awk '{print $1}' | sort | uniq -c

There are just 3 applications that have multiple cmdsynopsis. pgbench has 2
distinct tasks (load and run) so it makes sense to have 2 synopsis. pg_ctl has
multiple tasks and also deserves multiple synopsis. The complexity introduced
into vacuumdb (per table, per schema) seems to justify multiple synopsis too.
However, the same logic doesn't apply here. IMO 0002 shouldn't be applied
because the additional option (--all) doesn't justify multiple synopsis for
syntax clarification.

Yeah, also, vacuumdb doesn't have a separate line for --all in
synopsis. So, I agree with you about not adding '--all' option in the
synopsis.

--
With Regards,
Amit Kapila.

#103Amit Kapila
amit.kapila16@gmail.com
In reply to: Ashutosh Bapat (#98)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 5:30 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 5:54 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

*  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>

Most of this is unrelated to this patch. I suggest making a top-up
patch, we can commit it separately.

Isn't this documenting -a/--all option in synopsis. Why do you think
it's unrelated?

I was not clear at that stage whether to include it along with the
core patch, but today, I looked at it while responding to Euler and
found that it is not required. We can still discuss whether to include
it, but the main patch can be committed even without this.

--
With Regards,
Amit Kapila.

#104Amit Kapila
amit.kapila16@gmail.com
In reply to: Ashutosh Bapat (#97)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Tue, Mar 25, 2025 at 5:24 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Tue, Mar 25, 2025 at 5:15 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Tue, Mar 25, 2025 at 5:08 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

This looks mostly ready except the test changes. I believe when
committing, we are going to squash all three into a single commit. Is
that correct?

I would not prefer to commit 0003 as it is primarily because of test
+# run pg_createsubscriber with '--all' option without '--dry-run'. I
am not so sure whether it is really worth adding test cycles for this
option except for dry-run mode tests but we can discuss after
committing the core patch.

I am worried that without that test, we won't catch bugs in creating
slots, publications and subscriptions and thus causing problems in a
setup with --all.

The -all option internally maps to -d switches, so the current tests
with that option should suffice for the need you are expecting.

--
With Regards,
Amit Kapila.

#105Shubham Khanna
khannashubham1197@gmail.com
In reply to: Amit Kapila (#102)
4 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Mar 26, 2025 at 10:21 AM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Wed, Mar 26, 2025 at 2:05 AM Euler Taveira <euler@eulerto.com> wrote:

On Tue, Mar 25, 2025, at 8:07 AM, Shubham Khanna wrote:

Apologies for the oversight. I have attached the patches now. Please
find them included here.

I started looking at this patch. When I started reading the commit message, I
didn't understand why the options that provide names to objects are
incompatible with --all option. I agree that --all and --database should be
incompatible but the others shouldn't. We already have a check saying that the
number of specified objects cannot be different from the number of databases.
Why don't rely on it instead of forbidding these options?

We must ensure to match the order of objects specified with the
database as well (The doc says: The order of the multiple publication
name switches must match the order of database switches.). It is easy
for users to do that when explicitly specifying databases with -d
switches but not with --all case. I can see a way to extend the
current functionality to allow just fetching databases from the
publisher and displaying them to the user, then we can expect the user
to specify the names of other objects in the same order but not
otherwise.

What happens if you don't specify the dbname in -P option?

+   /* Establish a connection to the source server */
+   conn = connect_database(opt->pub_conninfo_str, true);

It defaults to user name. If you are using another superuser (such as admin)
the connection might fail if there is no database named 'admin'. vacuumdb that
has a similar --all option, uses another option (--maintenance-db) to discover
which databases should be vacuumed. I'm not suggesting to add the
--maintenance-db option. I wouldn't like to hardcode a specific database
(template1, for example) if there is no dbname in the connection string.
Instead, suggest the user to specify dbname into -P option. An error message
should be provided saying the exact reason: no dbname was specified.

But why shouldn't we use the same specification as vacuumdb, which is:
"If not specified, the postgres database will be used, or if that does
not exist, template1 will be used."?

I agree that ensuring the correct order of objects (like publications)
matching with databases is crucial, especially when explicitly
specifying databases using -d switches.
To address this, I have created a 0002 patch that aligns object
creation order with the corresponding databases.

I checked the applications that provide multiple synopsis using the following command.

grep '<cmdsynopsis' doc/src/sgml/ref/*.sgml | awk '{print $1}' | sort | uniq -c

There are just 3 applications that have multiple cmdsynopsis. pgbench has 2
distinct tasks (load and run) so it makes sense to have 2 synopsis. pg_ctl has
multiple tasks and also deserves multiple synopsis. The complexity introduced
into vacuumdb (per table, per schema) seems to justify multiple synopsis too.
However, the same logic doesn't apply here. IMO 0002 shouldn't be applied
because the additional option (--all) doesn't justify multiple synopsis for
syntax clarification.

Yeah, also, vacuumdb doesn't have a separate line for --all in
synopsis. So, I agree with you about not adding '--all' option in the
synopsis.

--

The attached patches contain the suggested changes. They also address
the comments provided by Ashutosh (at [1]/messages/by-id/CAExHW5uvp6LWfgcysohDaOaNhqAbmuc=9BwWke=6KPRZ+wVOkA@mail.gmail.com) and Euler (at [2]/messages/by-id/e32c358b-95b5-426c-9baa-281812821588@app.fastmail.com).

[1]: /messages/by-id/CAExHW5uvp6LWfgcysohDaOaNhqAbmuc=9BwWke=6KPRZ+wVOkA@mail.gmail.com
[2]: /messages/by-id/e32c358b-95b5-426c-9baa-281812821588@app.fastmail.com

Thanks and regards,
Shubham Khanna.

Attachments:

v22-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v22-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From a9ecbdbd8cd7457a60eb59e6dfc343a82b84dcf9 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v22] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     | 36 ++++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 84 ++++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              | 57 +++++++++++++
 3 files changed, 166 insertions(+), 11 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d011b79e5e6..7d105079127 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,23 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       Create one subscription on the target server for each non-template
+       database on the source server that allows connections, excluding
+       template databases or databases with connection restrictions.
+       Automatically generated names for subscriptions, publications, and
+       replication slots are used when this option is specified. This option
+       cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+       <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +111,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>-a</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +272,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +289,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +305,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index e2d6b7544bf..edf1b0ae119 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* all option */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void get_publisher_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +245,8 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all databases except template\n"
+			 "                                  databases or databases with connection restrictions\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1963,50 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * Fetch a list of all not-template databases from the source server.
+ * Internally, this is treated as if the user specified multiple --database
+ * options, one for each source database.
+ */
+static void
+get_publisher_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* Establish a connection to the source server */
+	conn = connectMaintenanceDatabase(opt->pub_conninfo_str, true);
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datconnlimit <> -2 ORDER BY 1");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2077,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2095,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2196,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2271,21 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 * This is treated as if the user specified multiple --database options,
+	 * one for each source database.
+	 */
+	if (opt.all_dbs)
+		get_publisher_databases(&opt);
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * If neither --database nor --all option is provided, try to obtain
+		 * the dbname from the publisher conninfo. If dbname parameter is not
+		 * available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f7a980ec799..a00a24eb4c1 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,63 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Verify that the required logical replication objects are output.
+# The expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v22-0004-Additional-test-cases.patchapplication/octet-stream; name=v22-0004-Additional-test-cases.patchDownload
From ae39409147da3f5ad1ca145a93608b83ab5c41d4 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Sat, 22 Mar 2025 19:08:30 +0530
Subject: [PATCH v22 4/4] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 151 ++++++++++++++++--
 1 file changed, 141 insertions(+), 10 deletions(-)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index a00a24eb4c1..a814a8b44d4 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -41,6 +41,25 @@ sub generate_db
 	return $dbname;
 }
 
+# Wait for subscriptions on the subscriber to catch up all changes.
+sub wait_for_all_subscriptions_caught_up
+{
+	my ($node_p, $node_s) = @_;
+
+	# Get subscription names
+	my $result = $node_s->safe_psql(
+		'postgres', qq(
+		SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+	));
+	my @subnames = split("\n", $result);
+
+	# Wait for all subscriptions to catch up
+	foreach my $subname (@subnames)
+	{
+		$node_p->wait_for_catchup($subname);
+	}
+}
+
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -386,6 +405,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +455,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -502,16 +572,7 @@ $result = $node_s->safe_psql(
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_s);
 
 # Confirm the failover slot has been removed
 $result = $node_s->safe_psql($db1,
@@ -537,10 +598,80 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+$node_s->stop;
+
+# Drop the database $db2 to verify subscriptions are handled correctly
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create a test table
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres',
+	"INSERT INTO tbl1 VALUES('row in database postgres')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+wait_for_all_subscriptions_caught_up($node_p, $node_u);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is( $result,
+	qq(row in database postgres),
+	"logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

v22-0003-Synopsis-for-all-option.patchapplication/octet-stream; name=v22-0003-Synopsis-for-all-option.patchDownload
From 519f23c2eb8051cafc7d013d326c48eea63757a4 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v22 3/3] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 7d105079127..d9556bf7354 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

v22-0002-Fetch-databases-from-publisher.patchapplication/octet-stream; name=v22-0002-Fetch-databases-from-publisher.patchDownload
From 102b4a7f0c165238688d9f5487592e648dfb6ef8 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Wed, 26 Mar 2025 13:22:23 +0530
Subject: [PATCH v22 2/2] Fetch databases from publisher

This patch allows current functionality of --all to fetch databases from the
publisher and display them to the user.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml   |  8 +++--
 src/bin/pg_basebackup/pg_createsubscriber.c | 34 ++++++++++++++++-----
 2 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 7d105079127..676dd933496 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -95,9 +95,11 @@ PostgreSQL documentation
        Create one subscription on the target server for each non-template
        database on the source server that allows connections, excluding
        template databases or databases with connection restrictions.
-       Automatically generated names for subscriptions, publications, and
-       replication slots are used when this option is specified. This option
-       cannot be used along with <option>--database</option>,
+       If database name is not specified in publisher-server, it will try to
+       connect with postgres/template1 database to fetch the databases from
+       primary. Automatically generated names for subscriptions, publications,
+       and replication slots are used when this option is specified.
+       This option cannot be used along with <option>--database</option>,
        <option>--publication</option>, <option>--replication-slot</option>, or
        <option>--subscription</option>.
       </para>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index edf1b0ae119..1a545d56007 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -125,7 +125,8 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
-static void get_publisher_databases(struct CreateSubscriberOptions *opt);
+static void get_publisher_databases(struct CreateSubscriberOptions *opt,
+									bool dbnamespecified);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -540,8 +541,9 @@ connect_database(const char *conninfo, bool exit_on_error)
 	conn = PQconnectdb(conninfo);
 	if (PQstatus(conn) != CONNECTION_OK)
 	{
-		pg_log_error("connection to database failed: %s",
-					 PQerrorMessage(conn));
+		if (exit_on_error)
+			pg_log_error("connection to database failed: %s",
+						 PQerrorMessage(conn));
 		PQfinish(conn);
 
 		if (exit_on_error)
@@ -1969,13 +1971,27 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
  * options, one for each source database.
  */
 static void
-get_publisher_databases(struct CreateSubscriberOptions *opt)
+get_publisher_databases(struct CreateSubscriberOptions *opt,
+						bool dbnamespecified)
 {
 	PGconn	   *conn;
 	PGresult   *res;
+	char	   *conninfo;
 
-	/* Establish a connection to the source server */
-	conn = connectMaintenanceDatabase(opt->pub_conninfo_str, true);
+	/* If a database name was specified, just connect to it. */
+	if (dbnamespecified)
+		conn = connect_database(opt->pub_conninfo_str, true);
+	else
+	{
+		/* Otherwise, try postgres first and then template1. */
+		conninfo = concat_conninfo_dbname(opt->pub_conninfo_str, "postgres");
+		conn = connect_database(conninfo, false);
+		if (!conn)
+		{
+			conninfo = concat_conninfo_dbname(opt->pub_conninfo_str, "template1");
+			conn = connect_database(conninfo, true);
+		}
+	}
 
 	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datconnlimit <> -2 ORDER BY 1");
 
@@ -2277,7 +2293,11 @@ main(int argc, char **argv)
 	 * one for each source database.
 	 */
 	if (opt.all_dbs)
-		get_publisher_databases(&opt);
+	{
+		bool		dbnamespecified = (dbname_conninfo != NULL);
+
+		get_publisher_databases(&opt, dbnamespecified);
+	}
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
-- 
2.41.0.windows.3

#106Hayato Kuroda (Fujitsu)
kuroda.hayato@fujitsu.com
In reply to: Shubham Khanna (#105)
RE: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Dear Shubham,

Thanks for updating the patch!

I agree that ensuring the correct order of objects (like publications)
matching with databases is crucial, especially when explicitly
specifying databases using -d switches.
To address this, I have created a 0002 patch that aligns object
creation order with the corresponding databases.

ISTM 0002 does completely different stuff. It handles the case when dbname is not
specified in -P. Commit message is also wrong.

Anyway, comments for 0001/0002:

01. connect_database
```
@@ -536,8 +541,9 @@ connect_database(const char *conninfo, bool exit_on_error)
        conn = PQconnectdb(conninfo);
        if (PQstatus(conn) != CONNECTION_OK)
        {
-               pg_log_error("connection to database failed: %s",
-                                        PQerrorMessage(conn));
+               if (exit_on_error)
+                       pg_log_error("connection to database failed: %s",
+                                                PQerrorMessage(conn));
                PQfinish(conn);
```

Any error messages should not be suppressed. I imagine you've added this because
this command may try to connect to the postgres/template1, but this change affects
all other parts. I feel this change is not needed.

02. main()
```
+       if (opt.all_dbs)
+       {
+               bool            dbnamespecified = (dbname_conninfo != NULL);
+
+               get_publisher_databases(&opt, dbnamespecified);
+       }
        if (opt.database_names.head == NULL)
```

Need a blank between them. Ashutosh pointed out [1]/messages/by-id/CAExHW5uvp6LWfgcysohDaOaNhqAbmuc=9BwWke=6KPRZ+wVOkA@mail.gmail.com because "else-if" was used in v21.
Now it becomes "if" again, the change must be reverted.

03. main()
```
-                * If --database option is not provided, try to obtain the dbname from
-                * the publisher conninfo. If dbname parameter is not available, error
-                * out.
+                * If neither --database nor --all option is provided, try to obtain
+                * the dbname from the publisher conninfo. If dbname parameter is not
+                * available, error out.
```

This comment must be updated because we can reach here even when -a is specified.
Personally, since the pg_log_info() describes the situation, it is enough;

Try to obtain the dbname from the publisher conninfo. If dbname parameter is not
available, error out.

04.
```
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+       [
+               'pg_createsubscriber',
+               '--verbose',
+               '--dry-run',
+               '--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+               '--pgdata' => $node_s->data_dir,
+               '--publisher-server' => $node_p->connstr($db1),
+               '--socketdir' => $node_s->host,
+               '--subscriber-port' => $node_s->port,
+               '--all',
+       ],
+       'run pg_createsubscriber with --all');
```

We should test the case when -P does not contain dbname. IIUC, it is enough to use
`node_p->connstr` instead of `node_p->connstr($db1)`.

[1]: /messages/by-id/CAExHW5uvp6LWfgcysohDaOaNhqAbmuc=9BwWke=6KPRZ+wVOkA@mail.gmail.com

Best regards,
Hayato Kuroda
FUJITSU LIMITED

#107Amit Kapila
amit.kapila16@gmail.com
In reply to: Shubham Khanna (#105)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Mar 26, 2025 at 4:02 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Let's combine 0001 and 0002.

A few minor comments:
*
+       If database name is not specified in publisher-server, it will try to
+       connect with postgres/template1 database to fetch the databases from
+       primary.

Can we change the above to: "If the database name is not specified in
publisher-server, the postgres database will be used, or if that does
not exist, template1 will be used."

*
+       Create one subscription on the target server for each non-template
+       database on the source server that allows connections, excluding
+       template databases or databases with connection restrictions.

The following text proposed by Euler seems better: "Create one
subscription per database on the target server. Exceptions are
template databases and databases that don't allow connections."

Please use this one at the above and other places where required.

--
With Regards,
Amit Kapila.

#108Shubham Khanna
khannashubham1197@gmail.com
In reply to: Amit Kapila (#107)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 27, 2025 at 12:16 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Wed, Mar 26, 2025 at 4:02 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

Let's combine 0001 and 0002.

Combined 0001 and 0002.

A few minor comments:
*
+       If database name is not specified in publisher-server, it will try to
+       connect with postgres/template1 database to fetch the databases from
+       primary.

Can we change the above to: "If the database name is not specified in
publisher-server, the postgres database will be used, or if that does
not exist, template1 will be used."

Fixed.

*
+       Create one subscription on the target server for each non-template
+       database on the source server that allows connections, excluding
+       template databases or databases with connection restrictions.

The following text proposed by Euler seems better: "Create one
subscription per database on the target server. Exceptions are
template databases and databases that don't allow connections."

Please use this one at the above and other places where required.

Fixed.

The attached patches contain the suggested changes. They also address
the comments provided by Kuroda-san (at [1]/messages/by-id/OSCPR01MB149660B6D00665F47896CA812F5A12@OSCPR01MB14966.jpnprd01.prod.outlook.com).

[1]: /messages/by-id/OSCPR01MB149660B6D00665F47896CA812F5A12@OSCPR01MB14966.jpnprd01.prod.outlook.com

Thanks and regards,
Shubham Khanna.

Attachments:

v23-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchapplication/octet-stream; name=v23-0001-Enhance-pg_createsubscriber-to-fetch-and-append-.patchDownload
From bac2815fb9502b38599971709a8e684ff3d38eb8 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Thu, 20 Mar 2025 13:52:27 +0530
Subject: [PATCH v23] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch enhances the 'pg_createsubscriber' utility by adding the '--all'
option. When '--all' is specified, the tool queries the source server
(publisher) for all databases and creates subscriptions on the target server
(subscriber) for databases with matching names.
This simplifies the process of converting a physical standby to a logical
subscriber, particularly during upgrades.

The options '--database', '--publication', '--subscription', and
'--replication-slot' cannot be used when '--all' is specified.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  37 ++++--
 src/bin/pg_basebackup/pg_createsubscriber.c   | 106 +++++++++++++++++-
 .../t/040_pg_createsubscriber.pl              |  57 ++++++++++
 3 files changed, 189 insertions(+), 11 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index d011b79e5e6..1f0ddd7f9f2 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -87,6 +87,24 @@ PostgreSQL documentation
    command-line arguments:
 
    <variablelist>
+    <varlistentry>
+     <term><option>-a</option></term>
+     <term><option>--all</option></term>
+     <listitem>
+      <para>
+       Create one subscription per database on the target server. Exceptions
+       are template databases and databases that don't allow connections.
+       If the database name is not specified in publisher-server, the postgres
+       database will be used, or if that does not exist, template1 will be used.
+       Automatically generated names for subscriptions, publications, and
+       replication slots are used when this option is specified.
+       This option cannot be used along with <option>--database</option>,
+       <option>--publication</option>, <option>--replication-slot</option>, or
+       <option>--subscription</option>.
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry>
      <term><option>-d <replaceable class="parameter">dbname</replaceable></option></term>
      <term><option>--database=<replaceable class="parameter">dbname</replaceable></option></term>
@@ -94,10 +112,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches. If <option>-d</option> option is not provided, the database
-       name will be obtained from <option>-P</option> option. If the database
-       name is not specified in either the <option>-d</option> option or
-       <option>-P</option> option, an error will be reported.
+       switches. This option cannot be used together with <option>-a</option>.
+       If <option>-d</option> option is not provided, the database name will be
+       obtained from <option>-P</option> option. If the database name is not
+       specified in either the <option>-d</option> option, or the
+       <option>-P</option> option, and <option>-a</option> option is not
+       specified, an error will be reported.
       </para>
      </listitem>
     </varlistentry>
@@ -253,7 +273,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple publication name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the publication name.
+       a generated name is assigned to the publication name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -269,7 +290,8 @@ PostgreSQL documentation
        otherwise an error is reported.  The order of the multiple replication
        slot name switches must match the order of database switches.  If this
        option is not specified, the subscription name is assigned to the
-       replication slot name.
+       replication slot name. This option cannot be used together with
+       <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
@@ -284,7 +306,8 @@ PostgreSQL documentation
        names must match the number of specified databases, otherwise an error
        is reported.  The order of the multiple subscription name switches must
        match the order of database switches.  If this option is not specified,
-       a generated name is assigned to the subscription name.
+       a generated name is assigned to the subscription name. This option cannot
+       be used together with <option>--all</option>.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index e2d6b7544bf..d0cff80d2b4 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -45,6 +45,7 @@ struct CreateSubscriberOptions
 	SimpleStringList sub_names; /* list of subscription names */
 	SimpleStringList replslot_names;	/* list of replication slot names */
 	int			recovery_timeout;	/* stop recovery after this time */
+	bool		all_dbs;		/* all option */
 	SimpleStringList objecttypes_to_remove; /* list of object types to remove */
 };
 
@@ -124,6 +125,8 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void get_publisher_databases(struct CreateSubscriberOptions *opt,
+									bool dbnamespecified);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -243,6 +246,8 @@ usage(void)
 	printf(_("Usage:\n"));
 	printf(_("  %s [OPTION]...\n"), progname);
 	printf(_("\nOptions:\n"));
+	printf(_("  -a, --all                       create subscriptions for all databases except template\n"
+			 "                                  databases or databases with connection restrictions\n"));
 	printf(_("  -d, --database=DBNAME           database in which to create a subscription\n"));
 	printf(_("  -D, --pgdata=DATADIR            location for the subscriber data directory\n"));
 	printf(_("  -n, --dry-run                   dry run, just show what would be done\n"));
@@ -1959,11 +1964,67 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/*
+ * Fetch a list of all not-template databases from the source server.
+ * Internally, this is treated as if the user specified multiple --database
+ * options, one for each source database.
+ */
+static void
+get_publisher_databases(struct CreateSubscriberOptions *opt,
+						bool dbnamespecified)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+
+	/* If a database name was specified, just connect to it. */
+	if (dbnamespecified)
+		conn = connect_database(opt->pub_conninfo_str, true);
+	else
+	{
+		/* Otherwise, try postgres first and then template1. */
+		char	   *conninfo;
+
+		conninfo = concat_conninfo_dbname(opt->pub_conninfo_str, "postgres");
+		conn = connect_database(conninfo, false);
+		pg_free(conninfo);
+		if (!conn)
+		{
+			conninfo = concat_conninfo_dbname(opt->pub_conninfo_str, "template1");
+			conn = connect_database(conninfo, true);
+			pg_free(conninfo);
+		}
+	}
+
+	res = PQexec(conn, "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datconnlimit <> -2 ORDER BY 1");
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("could not obtain a list of databases: %s", PQresultErrorMessage(res));
+		PQclear(res);
+		disconnect_database(conn, true);
+	}
+
+	for (int i = 0; i < PQntuples(res); i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+
+		/* Increment num_dbs to reflect multiple --database options */
+		num_dbs++;
+	}
+
+	PQclear(res);
+	disconnect_database(conn, false);
+}
+
 int
 main(int argc, char **argv)
 {
 	static struct option long_options[] =
 	{
+		{"all", no_argument, NULL, 'a'},
 		{"database", required_argument, NULL, 'd'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"dry-run", no_argument, NULL, 'n'},
@@ -2034,6 +2095,7 @@ main(int argc, char **argv)
 		0
 	};
 	opt.recovery_timeout = 0;
+	opt.all_dbs = false;
 
 	/*
 	 * Don't allow it to be run as root. It uses pg_ctl which does not allow
@@ -2051,11 +2113,14 @@ main(int argc, char **argv)
 
 	get_restricted_token();
 
-	while ((c = getopt_long(argc, argv, "d:D:np:P:R:s:t:TU:v",
+	while ((c = getopt_long(argc, argv, "ad:D:np:P:R:s:t:TU:v",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
+			case 'a':
+				opt.all_dbs = true;
+				break;
 			case 'd':
 				if (!simple_string_list_member(&opt.database_names, optarg))
 				{
@@ -2149,6 +2214,28 @@ main(int argc, char **argv)
 		}
 	}
 
+	/* Validate that --all is not used with incompatible options */
+	if (opt.all_dbs)
+	{
+		char	   *bad_switch = NULL;
+
+		if (num_dbs > 0)
+			bad_switch = "--database";
+		else if (num_pubs > 0)
+			bad_switch = "--publication";
+		else if (num_replslots > 0)
+			bad_switch = "--replication-slot";
+		else if (num_subs > 0)
+			bad_switch = "--subscription";
+
+		if (bad_switch)
+		{
+			pg_log_error("%s cannot be used with --all", bad_switch);
+			pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+			exit(1);
+		}
+	}
+
 	/* Any non-option arguments? */
 	if (optind < argc)
 	{
@@ -2202,14 +2289,25 @@ main(int argc, char **argv)
 	pg_log_info("validating subscriber connection string");
 	sub_base_conninfo = get_sub_conninfo(&opt);
 
+	/*
+	 * Fetch all databases from the source (publisher) if --all is specified.
+	 * This is treated as if the user specified multiple --database options,
+	 * one for each source database.
+	 */
+	if (opt.all_dbs)
+	{
+		bool		dbnamespecified = (dbname_conninfo != NULL);
+
+		get_publisher_databases(&opt, dbnamespecified);
+	}
+
 	if (opt.database_names.head == NULL)
 	{
 		pg_log_info("no database was specified");
 
 		/*
-		 * If --database option is not provided, try to obtain the dbname from
-		 * the publisher conninfo. If dbname parameter is not available, error
-		 * out.
+		 * Try to obtain the dbname from the publisher conninfo. If dbname
+		 * parameter is not available, error out.
 		 */
 		if (dbname_conninfo)
 		{
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index f7a980ec799..80153f7d77e 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -386,6 +386,63 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--database' and '--all' without '--dry-run'
+# and verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--database' => $db1,
+		'--all',
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
+# run pg_createsubscriber with '--publication' and '--all' and verify
+# the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--publication' => 'pub1',
+	],
+	qr/--publication cannot be used with --all/,
+	'fail if --publication is used with --all');
+
+# run pg_createsubscriber with '--all' option
+my ($stdout, $stderr) = run_command(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr,
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+# Verify that the required logical replication objects are output.
+# The expected count 3 refers to postgres, $db1 and $db2 databases.
+is(scalar(() = $stderr =~ /creating publication/g),
+	3, "verify publications are created for all databases");
+is(scalar(() = $stderr =~ /creating the replication slot/g),
+	3, "verify replication slots are created for all databases");
+is(scalar(() = $stderr =~ /creating subscription/g),
+	3, "verify subscriptions are created for all databases");
+
 # Run pg_createsubscriber on node S.  --verbose is used twice
 # to show more information.
 # In passing, also test the --enable-two-phase option and
-- 
2.41.0.windows.3

v23-0002-Synopsis-for-all-option.patchapplication/octet-stream; name=v23-0002-Synopsis-for-all-option.patchDownload
From 57429906eabf3f58d44b9076c431f6705110f140 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Fri, 21 Mar 2025 12:29:41 +0530
Subject: [PATCH v23 2/2] Synopsis for --all option

This patch contains the synopsis for the --all option.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 1f0ddd7f9f2..8d8574d7b5f 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -20,6 +20,27 @@ PostgreSQL documentation
  </refnamediv>
 
  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>
+   </group>
+  </cmdsynopsis>
+
   <cmdsynopsis>
    <command>pg_createsubscriber</command>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
-- 
2.41.0.windows.3

v23-0003-Additional-test-cases.patchapplication/octet-stream; name=v23-0003-Additional-test-cases.patchDownload
From 57dd646bb859183e2ca5791236334a3f33a45333 Mon Sep 17 00:00:00 2001
From: Khanna <Shubham.Khanna@fujitsu.com>
Date: Sat, 22 Mar 2025 19:08:30 +0530
Subject: [PATCH v23 3/3] Additional test cases

This patch contains the additional test cases related to the --all option.
---
 .../t/040_pg_createsubscriber.pl              | 151 ++++++++++++++++--
 1 file changed, 141 insertions(+), 10 deletions(-)

diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index a00a24eb4c1..a814a8b44d4 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -41,6 +41,25 @@ sub generate_db
 	return $dbname;
 }
 
+# Wait for subscriptions on the subscriber to catch up all changes.
+sub wait_for_all_subscriptions_caught_up
+{
+	my ($node_p, $node_s) = @_;
+
+	# Get subscription names
+	my $result = $node_s->safe_psql(
+		'postgres', qq(
+		SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
+	));
+	my @subnames = split("\n", $result);
+
+	# Wait for all subscriptions to catch up
+	foreach my $subname (@subnames)
+	{
+		$node_p->wait_for_catchup($subname);
+	}
+}
+
 #
 # Test mandatory options
 command_fails(['pg_createsubscriber'],
@@ -386,6 +405,23 @@ command_ok(
 	],
 	'run pg_createsubscriber without --databases');
 
+# run pg_createsubscriber with '--all' and '--database' and verify the
+# failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--database' => $db1,
+	],
+	qr/--database cannot be used with --all/,
+	'fail if --database is used with --all');
+
 # run pg_createsubscriber with '--database' and '--all' without '--dry-run'
 # and verify the failure
 command_fails_like(
@@ -419,6 +455,40 @@ command_fails_like(
 	qr/--publication cannot be used with --all/,
 	'fail if --publication is used with --all');
 
+# run pg_createsubscriber with '--replication-slot' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--replication-slot' => 'replslot1',
+		'--all',
+	],
+	qr/--replication-slot cannot be used with --all/,
+	'fail if --replication-slot is used with --all');
+
+# run pg_createsubscriber with '--subscription' and '--all' and
+# verify the failure
+command_fails_like(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--dry-run',
+		'--pgdata' => $node_s->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_s->host,
+		'--subscriber-port' => $node_s->port,
+		'--all',
+		'--subscription' => 'sub1',
+	],
+	qr/--subscription cannot be used with --all/,
+	'fail if --subscription is used with --all');
+
 # run pg_createsubscriber with '--all' option
 my ($stdout, $stderr) = run_command(
 	[
@@ -502,16 +572,7 @@ $result = $node_s->safe_psql(
 ));
 is($result, qq(0), 'pre-existing subscription was dropped');
 
-# Get subscription names
-$result = $node_s->safe_psql(
-	'postgres', qq(
-	SELECT subname FROM pg_subscription WHERE subname ~ '^pg_createsubscriber_'
-));
-my @subnames = split("\n", $result);
-
-# Wait subscriber to catch up
-$node_s->wait_for_subscription_sync($node_p, $subnames[0]);
-$node_s->wait_for_subscription_sync($node_p, $subnames[1]);
+wait_for_all_subscriptions_caught_up($node_p, $node_s);
 
 # Confirm the failover slot has been removed
 $result = $node_s->safe_psql($db1,
@@ -537,10 +598,80 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+$node_s->stop;
+
+# Drop the database $db2 to verify subscriptions are handled correctly
+$node_p->safe_psql('postgres', "DROP DATABASE \"$db2\"");
+
+# On node P create a test table
+$node_p->safe_psql('postgres', 'CREATE TABLE tbl1 (a text)');
+
+# Set up node U as standby linking to node P
+$node_p->backup('backup_3');
+my $node_u = PostgreSQL::Test::Cluster->new('node_u');
+$node_u->init_from_backup($node_p, 'backup_3', has_streaming => 1);
+$node_u->set_standby_mode();
+
+# run pg_createsubscriber with '--all' option without '--dry-run'
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--recovery-timeout' => $PostgreSQL::Test::Utils::timeout_default,
+		'--pgdata' => $node_u->data_dir,
+		'--publisher-server' => $node_p->connstr($db1),
+		'--socketdir' => $node_u->host,
+		'--subscriber-port' => $node_u->port,
+		'--all',
+	],
+	'run pg_createsubscriber with --all');
+
+$node_u->start;
+
+# Verify that user databases (postgres, $db1) got subscriptions.
+$result = $node_u->safe_psql(
+	'postgres',
+	'SELECT datname FROM pg_subscription,
+	pg_database WHERE subdbid = pg_database.oid and datistemplate = \'f\' ORDER BY pg_database.oid'
+);
+is( $result, "postgres
+$db1", 'subscription is created on the required databases');
+
+# Verify template databases do not have subscriptions
+$result = $node_u->safe_psql(
+	'postgres',
+	"SELECT count(*) FROM pg_subscription, pg_database
+	 WHERE subdbid = pg_database.oid and datistemplate = 't';"
+);
+is($result, '0', 'subscription is not created on template databases');
+
+# Verify logical replication works for all databases
+# Insert rows on node P
+$node_p->safe_psql('postgres',
+	"INSERT INTO tbl1 VALUES('row in database postgres')");
+$node_p->safe_psql($db1, "INSERT INTO tbl1 VALUES('fourth row')");
+
+wait_for_all_subscriptions_caught_up($node_p, $node_u);
+
+# Check result in database 'postgres' of node U
+$result = $node_u->safe_psql('postgres', 'SELECT * FROM tbl1');
+is( $result,
+	qq(row in database postgres),
+	"logical replication works in database postgres");
+
+# Check result in database $db1 of node U
+$result = $node_u->safe_psql($db1, 'SELECT * FROM tbl1');
+is( $result, qq(first row
+second row
+third row
+fourth row),
+	"logical replication works in database $db1");
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
+$node_u->teardown_node;
 $node_f->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3

#109Amit Kapila
amit.kapila16@gmail.com
In reply to: Shubham Khanna (#108)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Mar 27, 2025 at 1:35 PM Shubham Khanna
<khannashubham1197@gmail.com> wrote:

The attached patches contain the suggested changes. They also address
the comments provided by Kuroda-san (at [1]).

Pushed after some cosmetic changes.

--
With Regards,
Amit Kapila.

#110Peter Smith
smithpb2250@gmail.com
In reply to: Amit Kapila (#109)
3 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

Hi,

I was looking at the recent push for the pg_createsubscription "--all"
option [1]https://github.com/postgres/postgres/commit/fb2ea12f42b9453853be043b8ed107e136e1ccb7, because I was absent for several weeks prior.

I found some minor non-functional issues as follows:

======
doc/src/sgml/ref/pg_createsubscriber.sgml

1.
+       If the database name is not specified in publisher-server, the postgres
+       database will be used, or if that does not exist, template1
will be used.

1a.
There is missing SGML markup in the description of "--all"

- "publisher-server" ==> "the <option>--publisher-server</option>
connection string"
- "postgres database" ==> "<literal>postgres</literal> database"
- "template1 database" ==> "<literal>template1</literal> database"

~

1b.
That sentence "If the database name ..." doesn't give any context
about what it is referring to. I think it needs to be worded more
similarly to the vacuum --maintenance-db option [2]https://www.postgresql.org/docs/current/app-vacuumdb.html to say it is
connecting to this database in order to "discover" the list of
databases.

SUGGESTION
To discover the list of all databases, connect to the source server
using the database name specified in the
<option>--publisher-server</option> connection string, or if not
specified, the <literal>postgres</literal> database will be used, or
if that does not exist, <literal>template1</literal> will be used.

~

1c.
I didn't think the sentence quite belongs here because it isn't
describing anything about the --all option. Instead, it is describing
how the source server connection is made. So, I felt this might be
better put later in the "Notes/Prerequisites" section where other
source server connection information is described.

======
src/bin/pg_basebackup/pg_createsubscriber.c

get_publisher_databases:

2.
+/*
+ * Fetch a list of all not-template databases from the source server.
+ * Internally, this is treated as if the user specified multiple --database
+ * options, one for each source database.
+ */

/not-template/connectable non-template/

======

Patches are attached to implement the above.

0001. Docs some fixes to the --all description. (see #1a and #1b)
0002. Docs moves part of the --all description to the
Note/Prerequisites section. (see #1c)
0003. Fixes to code comments. (see #2)

======
[1]: https://github.com/postgres/postgres/commit/fb2ea12f42b9453853be043b8ed107e136e1ccb7
[2]: https://www.postgresql.org/docs/current/app-vacuumdb.html

Kind Regards,
Peter Smith.
Fujitsu Australia

Attachments:

v20250409-0001-DOCS-change-markup-and-wording-for-the-all.patchapplication/octet-stream; name=v20250409-0001-DOCS-change-markup-and-wording-for-the-all.patchDownload
From 21786438d2a7b1d159afcc076bdc7d031420d185 Mon Sep 17 00:00:00 2001
From: Peter Smith <peter.b.smith@fujitsu.com>
Date: Wed, 9 Apr 2025 09:47:04 +1000
Subject: [PATCH v20250409] DOCS - change markup and wording for the --all
 option description

---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 73377aa..4b1d08d 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -94,8 +94,11 @@ PostgreSQL documentation
       <para>
        Create one subscription per database on the target server. Exceptions
        are template databases and databases that don't allow connections.
-       If the database name is not specified in publisher-server, the postgres
-       database will be used, or if that does not exist, template1 will be used.
+       To discover the list of all databases, connect to the source server
+       using the database name specified in the <option>--publisher-server</option>
+       connection string, or if not specified, the <literal>postgres</literal>
+       database will be used, or if that does not exist, <literal>template1</literal>
+       will be used.
        Automatically generated names for subscriptions, publications, and
        replication slots are used when this option is specified.
        This option cannot be used along with <option>--database</option>,
-- 
1.8.3.1

v20250409-0003-Minor-comment-fix.patchapplication/octet-stream; name=v20250409-0003-Minor-comment-fix.patchDownload
From 0df9521cb4b2a30ab81fe846d524c7ef5740778b Mon Sep 17 00:00:00 2001
From: Peter Smith <peter.b.smith@fujitsu.com>
Date: Wed, 9 Apr 2025 11:31:56 +1000
Subject: [PATCH v20250409] Minor comment fix

---
 src/bin/pg_basebackup/pg_createsubscriber.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 1d2f39e..42aaa42 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -1963,8 +1963,8 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 }
 
 /*
- * Fetch a list of all not-template databases from the source server and form
- * a list such that they appear as if the user has specified multiple
+ * Fetch a list of all connectable non-template databases from the source server
+ * and form a list such that they appear as if the user has specified multiple
  * --database options, one for each source database.
  */
 static void
-- 
1.8.3.1

v20250409-0002-DOCS-move-the-description-how-the-all-conn.patchapplication/octet-stream; name=v20250409-0002-DOCS-move-the-description-how-the-all-conn.patchDownload
From e58014ed33453644f7d37caea0096bf7bc67f1a7 Mon Sep 17 00:00:00 2001
From: Peter Smith <peter.b.smith@fujitsu.com>
Date: Wed, 9 Apr 2025 10:35:23 +1000
Subject: [PATCH v20250409] DOCS - move the description how the --all
 connection is made

---
 doc/src/sgml/ref/pg_createsubscriber.sgml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 4b1d08d..e30a144 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -94,11 +94,6 @@ PostgreSQL documentation
       <para>
        Create one subscription per database on the target server. Exceptions
        are template databases and databases that don't allow connections.
-       To discover the list of all databases, connect to the source server
-       using the database name specified in the <option>--publisher-server</option>
-       connection string, or if not specified, the <literal>postgres</literal>
-       database will be used, or if that does not exist, <literal>template1</literal>
-       will be used.
        Automatically generated names for subscriptions, publications, and
        replication slots are used when this option is specified.
        This option cannot be used along with <option>--database</option>,
@@ -386,6 +381,11 @@ PostgreSQL documentation
     replication slots.  The source server must have <xref
     linkend="guc-max-wal-senders"/> configured to a value greater than or equal
     to the number of specified databases and existing WAL sender processes.
+    When discovering the list of all databases for the <option>--all</option>
+    option, connect to the source server using the database name specified in
+    the <option>--publisher-server</option> connection string, or if not specified,
+    the <literal>postgres</literal> database will be used, or if that does not exist,
+    <literal>template1</literal> will be used.
    </para>
   </refsect2>
 
-- 
1.8.3.1

#111Peter Smith
smithpb2250@gmail.com
In reply to: Amit Kapila (#103)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Mar 26, 2025 at 3:54 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Tue, Mar 25, 2025 at 5:30 PM Ashutosh Bapat
<ashutosh.bapat.oss@gmail.com> wrote:

On Thu, Mar 20, 2025 at 5:54 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

*  <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_createsubscriber</command>
+   <arg rep="repeat"><replaceable>option</replaceable></arg>
+   <group choice="plain">
+    <group choice="req">
+     <arg choice="plain"><option>-a</option></arg>
+     <arg choice="plain"><option>--all</option></arg>
+    </group>
+    <group choice="req">
+     <arg choice="plain"><option>-D</option> </arg>
+     <arg choice="plain"><option>--pgdata</option></arg>
+    </group>
+    <replaceable>datadir</replaceable>
+    <group choice="req">
+     <arg choice="plain"><option>-P</option></arg>
+     <arg choice="plain"><option>--publisher-server</option></arg>
+    </group>
+    <replaceable>connstr</replaceable>

Most of this is unrelated to this patch. I suggest making a top-up
patch, we can commit it separately.

Isn't this documenting -a/--all option in synopsis. Why do you think
it's unrelated?

I was not clear at that stage whether to include it along with the
core patch, but today, I looked at it while responding to Euler and
found that it is not required. We can still discuss whether to include
it, but the main patch can be committed even without this.

I've created a separate thread [1]/messages/by-id/CAHut+PueY_DyuBy6SuvJev2DWJVGtg=9WG9WXvYQDJu39gV6TQ@mail.gmail.com to continue the discussion about
the synopsis.

======
[1]: /messages/by-id/CAHut+PueY_DyuBy6SuvJev2DWJVGtg=9WG9WXvYQDJu39gV6TQ@mail.gmail.com

Kind Regards,
Peter Smith.
Fujitsu Australia

#112Amit Kapila
amit.kapila16@gmail.com
In reply to: Peter Smith (#110)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Apr 9, 2025 at 7:48 AM Peter Smith <smithpb2250@gmail.com> wrote:

I was looking at the recent push for the pg_createsubscription "--all"
option [1], because I was absent for several weeks prior.

I found some minor non-functional issues as follows:

======
doc/src/sgml/ref/pg_createsubscriber.sgml
~

1c.
I didn't think the sentence quite belongs here because it isn't
describing anything about the --all option. Instead, it is describing
how the source server connection is made.

This sentence describes how we connect when --all option is used. For
other cases, see the description in the -d option. So, this change is
not required. If you agree, then you can combine the other two
patches, and we can proceed with those.

--
With Regards,
Amit Kapila.

#113Peter Smith
smithpb2250@gmail.com
In reply to: Amit Kapila (#112)
1 attachment(s)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Wed, Apr 9, 2025 at 8:08 PM Amit Kapila <amit.kapila16@gmail.com> wrote:

On Wed, Apr 9, 2025 at 7:48 AM Peter Smith <smithpb2250@gmail.com> wrote:

I was looking at the recent push for the pg_createsubscription "--all"
option [1], because I was absent for several weeks prior.

I found some minor non-functional issues as follows:

======
doc/src/sgml/ref/pg_createsubscriber.sgml
~

1c.
I didn't think the sentence quite belongs here because it isn't
describing anything about the --all option. Instead, it is describing
how the source server connection is made.

This sentence describes how we connect when --all option is used. For
other cases, see the description in the -d option. So, this change is
not required. If you agree, then you can combine the other two
patches, and we can proceed with those.

OK. Here is a single patch which combines previous patches 0001 and 0003.

======
Kind Regards,
Peter Smith.
Fujitsu Australia

Attachments:

v20250410-0001-Fix-DOCS-and-code-comment-for-option-all.patchapplication/octet-stream; name=v20250410-0001-Fix-DOCS-and-code-comment-for-option-all.patchDownload
From 9fcc11b8c470d15a9f01c9266fedf2f7ca058a1d Mon Sep 17 00:00:00 2001
From: Peter Smith <peter.b.smith@fujitsu.com>
Date: Thu, 10 Apr 2025 09:39:45 +1000
Subject: [PATCH v20250410] Fix DOCS and code comment for option --all

---
 doc/src/sgml/ref/pg_createsubscriber.sgml   | 7 +++++--
 src/bin/pg_basebackup/pg_createsubscriber.c | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 73377aa..4b1d08d 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -94,8 +94,11 @@ PostgreSQL documentation
       <para>
        Create one subscription per database on the target server. Exceptions
        are template databases and databases that don't allow connections.
-       If the database name is not specified in publisher-server, the postgres
-       database will be used, or if that does not exist, template1 will be used.
+       To discover the list of all databases, connect to the source server
+       using the database name specified in the <option>--publisher-server</option>
+       connection string, or if not specified, the <literal>postgres</literal>
+       database will be used, or if that does not exist, <literal>template1</literal>
+       will be used.
        Automatically generated names for subscriptions, publications, and
        replication slots are used when this option is specified.
        This option cannot be used along with <option>--database</option>,
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index 1d2f39e..42aaa42 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -1963,8 +1963,8 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 }
 
 /*
- * Fetch a list of all not-template databases from the source server and form
- * a list such that they appear as if the user has specified multiple
+ * Fetch a list of all connectable non-template databases from the source server
+ * and form a list such that they appear as if the user has specified multiple
  * --database options, one for each source database.
  */
 static void
-- 
1.8.3.1

#114Amit Kapila
amit.kapila16@gmail.com
In reply to: Peter Smith (#113)
Re: Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.

On Thu, Apr 10, 2025 at 5:22 AM Peter Smith <smithpb2250@gmail.com> wrote:

OK. Here is a single patch which combines previous patches 0001 and 0003.

Pushed.

--
With Regards,
Amit Kapila.