From 0c869f522875eae51483ab530b0884f2975bb2cd Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Thu, 4 Aug 2022 11:17:00 +0900 Subject: [PATCH v4] Add wait_for_subscription_sync for TAP tests. The TAP tests for logical replication in src/test/subscription are using the following code in many places to make sure that the subscription is synchronized with the publisher: $node_publisher->wait_for_catchup('tap_sub'); $node_subscriber->poll_query_until('postgres', qq[SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's')]); The new function wait_for_subscription_sync() can be used to replace the above code. This eliminates duplicated code and makes it easier to write future tests. Author: Masahiko Sawada Reviewed by: Amit Kapila, Shi yu Discussion: https://postgr.es/m/CAD21AoC-fvAkaKHa4t1urupwL8xbAcWRePeETvshvy80f6WV1A@mail.gmail.com --- src/test/perl/PostgresNode.pm | 42 ++++++++++++++++++++++ src/test/subscription/t/001_rep_changes.pl | 5 +-- src/test/subscription/t/002_types.pl | 5 +-- src/test/subscription/t/004_sync.pl | 14 +++----- src/test/subscription/t/005_encoding.pl | 9 ++--- src/test/subscription/t/006_rewrite.pl | 9 ++--- src/test/subscription/t/008_diff_schema.pl | 12 ++----- src/test/subscription/t/100_bugs.pl | 9 ++--- 8 files changed, 57 insertions(+), 48 deletions(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 60d24bf8c0..3d87331561 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -1942,6 +1942,48 @@ qq[SELECT '$target_lsn' <= ${mode}_lsn FROM pg_catalog.pg_replication_slots WHER print "done\n"; } +=item $node->wait_for_subscription_sync(publisher, subname, dbname) + +Wait for all tables in pg_subscription_rel to complete the initial +synchronization (i.e to be either in 'syncdone' or 'ready' state). + +If the publisher node is given, additionally, check if the subscriber has +caught up to what has been committed on the primary. This is useful to +ensure that the initial data synchronization has been completed after +creating a new subscription. + +If there is no active replication connection from this peer, wait until +poll_query_until timeout. + +This is not a test. It die()s on failure. + +=cut + +sub wait_for_subscription_sync +{ + my ($self, $publisher, $subname, $dbname) = @_; + my $name = $self->name; + + $dbname = defined($dbname) ? $dbname : 'postgres'; + + # Wait for all tables to finish initial sync. + print "Waiting for all subscriptions in \"$name\" to synchronize data\n"; + my $query = + qq[SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');]; + $self->poll_query_until($dbname, $query) + or croak "timed out waiting for subscriber to synchronize data"; + + # Then, wait for the replication to catchup if required. + if (defined($publisher)) + { + croak 'subscription name must be specified' unless defined($subname); + $publisher->wait_for_catchup($subname, 'replay', $publisher->lsn('write')); + } + + print "done\n"; + return; +} + =pod =item $node->wait_for_log(regexp, offset) diff --git a/src/test/subscription/t/001_rep_changes.pl b/src/test/subscription/t/001_rep_changes.pl index 5decd8d759..f3ed44246e 100644 --- a/src/test/subscription/t/001_rep_changes.pl +++ b/src/test/subscription/t/001_rep_changes.pl @@ -85,10 +85,7 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) or die "Timed out while waiting for subscriber to catch up"; # Also wait for initial table sync to finish -my $synced_query = -"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; my $result = $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); diff --git a/src/test/subscription/t/002_types.pl b/src/test/subscription/t/002_types.pl index 1c486bf44a..f813953dd1 100644 --- a/src/test/subscription/t/002_types.pl +++ b/src/test/subscription/t/002_types.pl @@ -119,10 +119,7 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) or die "Timed out while waiting for subscriber to catch up"; # Wait for initial sync to finish as well -my $synced_query = -"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('s', 'r');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; # Insert initial test data $node_publisher->safe_psql( diff --git a/src/test/subscription/t/004_sync.pl b/src/test/subscription/t/004_sync.pl index 05fd2f0e6c..39420f2626 100644 --- a/src/test/subscription/t/004_sync.pl +++ b/src/test/subscription/t/004_sync.pl @@ -44,10 +44,7 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) or die "Timed out while waiting for subscriber to catch up"; # Also wait for initial table sync to finish -my $synced_query = -"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; my $result = $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep"); @@ -73,8 +70,7 @@ $node_subscriber->poll_query_until('postgres', $started_query) $node_subscriber->safe_psql('postgres', "DELETE FROM tab_rep;"); # wait for sync to finish this time -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; # check that all data is synced $result = @@ -109,8 +105,7 @@ $node_subscriber->safe_psql('postgres', ); # and wait for data sync to finish again -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; # check that all data is synced $result = @@ -137,8 +132,7 @@ $node_subscriber->safe_psql('postgres', "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION"); # wait for sync to finish -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; $result = $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep_next"); diff --git a/src/test/subscription/t/005_encoding.pl b/src/test/subscription/t/005_encoding.pl index 2b0c47c07d..dc46594094 100644 --- a/src/test/subscription/t/005_encoding.pl +++ b/src/test/subscription/t/005_encoding.pl @@ -39,13 +39,8 @@ $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION mysub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION mypub;" ); -wait_for_caught_up($node_publisher, $appname); - -# Wait for initial sync to finish as well -my $synced_query = - "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('s', 'r');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +# Wait for initial sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, $appname); $node_publisher->safe_psql('postgres', q{INSERT INTO test1 VALUES (1, E'Mot\xc3\xb6rhead')}); # hand-rolled UTF-8 diff --git a/src/test/subscription/t/006_rewrite.pl b/src/test/subscription/t/006_rewrite.pl index 20151b25ec..3bdd053b42 100644 --- a/src/test/subscription/t/006_rewrite.pl +++ b/src/test/subscription/t/006_rewrite.pl @@ -42,13 +42,8 @@ $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION mysub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION mypub;" ); -wait_for_caught_up($node_publisher, $appname); - -# Wait for initial sync to finish as well -my $synced_query = - "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('s', 'r');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +# Wait for initial sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, $appname); $node_publisher->safe_psql('postgres', q{INSERT INTO test1 (a, b) VALUES (1, 'one'), (2, 'two');}); diff --git a/src/test/subscription/t/008_diff_schema.pl b/src/test/subscription/t/008_diff_schema.pl index 580c56743f..259cd49ac2 100644 --- a/src/test/subscription/t/008_diff_schema.pl +++ b/src/test/subscription/t/008_diff_schema.pl @@ -42,13 +42,8 @@ $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub" ); -wait_for_caught_up($node_publisher, $appname); - -# Also wait for initial table sync to finish -my $synced_query = -"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +# Wait for initial table sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, $appname); my $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); @@ -102,8 +97,7 @@ $node_subscriber->safe_psql('postgres', $node_subscriber->safe_psql('postgres', "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION"); -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +$node_subscriber->wait_for_subscription_sync; # Add replica identity column. (The serial is not necessary, but it's # a convenient way to get a default on the new column so that rows diff --git a/src/test/subscription/t/100_bugs.pl b/src/test/subscription/t/100_bugs.pl index f0d3134dcb..fa1ad13507 100644 --- a/src/test/subscription/t/100_bugs.pl +++ b/src/test/subscription/t/100_bugs.pl @@ -161,13 +161,8 @@ $node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=tap_sub' PUBLICATION tap_pub" ); -wait_for_caught_up($node_publisher, 'tap_sub'); - -# Also wait for initial table sync to finish -my $synced_query = -"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('s', 'r');"; -$node_subscriber->poll_query_until('postgres', $synced_query) - or die "Timed out while waiting for subscriber to synchronize data"; +# Wait for initial table sync to finish +$node_subscriber->wait_for_subscription_sync($node_publisher, 'tap_sub'); is( $node_subscriber->safe_psql( 'postgres', "SELECT * FROM tab_replidentity_index"), -- 2.24.3 (Apple Git-128)