standby recovery fails (tablespace related) (tentative patch and discussion)

diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..0bc63f48da 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -522,6 +522,44 @@ XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 	return buffer;
 }
 
+/*
+ * XLogMakePGDirectory
+ *
+ * There is a possibility that WAL replay causes a creation of the same
+ * directory left by the previous crash. Issuing ERROR prevents the caller
+ * from continuing recovery.
+ *
+ * To prevent that case, this function issues WARNING instead of ERROR on
+ * error if consistency is not reached yet.
+ */
+int
+XLogMakePGDirectory(const char *directoryName)
+{
+	int ret;
+
+	ret = MakePGDirectory(directoryName);
+
+	if (ret != 0)
+	{
+		int elevel = ERROR;
+
+		/*
+		 * We might get error trying to create existing directory that is to
+		 * be removed just after.  Don't issue ERROR in the case. Recovery
+		 * will stop if we again failed after reaching consistency.
+		 */
+		if (InRecovery && !reachedConsistency)
+			elevel = WARNING;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not create directory \"%s\": %m", directoryName)));
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * Struct actually returned by XLogFakeRelcacheEntry, though the declared
  * return type is Relation.
diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c
index 30f6200a86..0216270dd3 100644
--- a/src/backend/storage/file/copydir.c
+++ b/src/backend/storage/file/copydir.c
@@ -22,11 +22,11 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/xlogutils.h"
 #include "storage/copydir.h"
 #include "storage/fd.h"
 #include "miscadmin.h"
 #include "pgstat.h"
-
 /*
  * copydir: copy a directory
  *
@@ -41,10 +41,12 @@ copydir(char *fromdir, char *todir, bool recurse)
 	char		fromfile[MAXPGPATH * 2];
 	char		tofile[MAXPGPATH * 2];
 
-	if (MakePGDirectory(todir) != 0)
-		ereport(ERROR,
-				(errcode_for_file_access(),
-				 errmsg("could not create directory \"%s\": %m", todir)));
+	/*
+	 * We might have to skip copydir to continue recovery. See the function
+	 * for details.
+	 */
+	if (XLogMakePGDirectory(todir) != 0)
+		return;
 
 	xldir = AllocateDir(fromdir);
 
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0ab5ba62f5..46a7596315 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -43,6 +43,7 @@ extern XLogRedoAction XLogReadBufferForRedoExtended(XLogReaderState *record,
 
 extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 					   BlockNumber blkno, ReadBufferMode mode);
+extern int XLogMakePGDirectory(const char *directoryName);
 
 extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
 extern void FreeFakeRelcacheEntry(Relation fakerel);

diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..01331f0da9 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -522,6 +522,44 @@ XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 	return buffer;
 }
 
+/*
+ * XLogMakePGDirectory
+ *
+ * There is a possibility that WAL replay causes an error by creation of a
+ * directory under a directory removed before the previous crash. Issuing
+ * ERROR prevents the caller from continuing recovery.
+ *
+ * To prevent that case, this function issues WARNING instead of ERROR on
+ * error if consistency is not reached yet.
+ */
+int
+XLogMakePGDirectory(const char *directoryName)
+{
+	int ret;
+
+	ret = MakePGDirectory(directoryName);
+
+	if (ret != 0)
+	{
+		int elevel = ERROR;
+
+		/*
+		 * We might get error trying to create existing directory that is to
+		 * be removed just after.  Don't issue ERROR in the case. Recovery
+		 * will stop if we again failed after reaching consistency.
+		 */
+		if (InRecovery && !reachedConsistency)
+			elevel = WARNING;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not create directory \"%s\": %m", directoryName)));
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * Struct actually returned by XLogFakeRelcacheEntry, though the declared
  * return type is Relation.
diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c
index 30f6200a86..0216270dd3 100644
--- a/src/backend/storage/file/copydir.c
+++ b/src/backend/storage/file/copydir.c
@@ -22,11 +22,11 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/xlogutils.h"
 #include "storage/copydir.h"
 #include "storage/fd.h"
 #include "miscadmin.h"
 #include "pgstat.h"
-
 /*
  * copydir: copy a directory
  *
@@ -41,10 +41,12 @@ copydir(char *fromdir, char *todir, bool recurse)
 	char		fromfile[MAXPGPATH * 2];
 	char		tofile[MAXPGPATH * 2];
 
-	if (MakePGDirectory(todir) != 0)
-		ereport(ERROR,
-				(errcode_for_file_access(),
-				 errmsg("could not create directory \"%s\": %m", todir)));
+	/*
+	 * We might have to skip copydir to continue recovery. See the function
+	 * for details.
+	 */
+	if (XLogMakePGDirectory(todir) != 0)
+		return;
 
 	xldir = AllocateDir(fromdir);
 
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0ab5ba62f5..46a7596315 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -43,6 +43,7 @@ extern XLogRedoAction XLogReadBufferForRedoExtended(XLogReaderState *record,
 
 extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 					   BlockNumber blkno, ReadBufferMode mode);
+extern int XLogMakePGDirectory(const char *directoryName);
 
 extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
 extern void FreeFakeRelcacheEntry(Relation fakerel);

From dbe6306a730f94a5bd8beaf0e534c28ebdd815d4 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Mon, 22 Apr 2019 20:10:20 +0900
Subject: [PATCH 1/2] Allow TAP test to excecise tablespace.

To perform tablespace related checks, this patch lets
PostgresNode::backup have a new parameter "tablespace_mapping", and
make init_from_backup handle capable to handle a backup created using
tablespace_mapping.
---
 src/test/perl/PostgresNode.pm  | 10 ++++++++--
 src/test/perl/RecursiveCopy.pm | 33 +++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 76874141c5..59a939821d 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -540,13 +540,19 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mapping})
+	{
+		push(@rest, "--tablespace-mapping=$params{tablespace_mapping}");
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..c912ce412d 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::real_dir($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
-- 
2.16.3

From 382910fbe3738c9098c0568cdc992928f471c7c5 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Mon, 22 Apr 2019 20:10:25 +0900
Subject: [PATCH 2/2] Add check for recovery failure caused by tablespace.

Removal of a tablespace on master can cause recovery failure on
standby. This patch adds the check for the case.
---
 src/test/recovery/t/011_crash_recovery.pl | 52 ++++++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 5dc52412ca..d1eb9edccf 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 4;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +66,53 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+
+# Ensure that tablespace removal doesn't cause error while recoverying
+# the preceding create datbase or objects.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $tspDir_master = TestLib::tempdir;
+my $realTSDir_master = TestLib::real_dir($tspDir_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master'");
+
+my $tspDir_standby = TestLib::tempdir;
+my $realTSDir_standby = TestLib::real_dir($tspDir_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mapping =>
+					   "$realTSDir_master=$realTSDir_standby");
+my $node_standby = get_new_node('standby');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This leaves a CREATE DATBASE WAL record
+# that is to be applied to already-removed tablespace.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE db1 WITH TABLESPACE ts1;
+						  DROP DATABASE db1;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
-- 
2.16.3

From 5e3a9b682e6ec2b6cb4e019409112687bd598ebc Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Mon, 22 Apr 2019 20:59:15 +0900
Subject: [PATCH 3/3] Fix failure of standby startup caused by tablespace
 removal

When standby restarts after a crash after drop of a tablespace,
there's a possibility that recovery fails trying an object-creation in
already removed tablespace directory. Allow recovery to continue by
ignoring the error if not reaching consistency point.
---
 src/backend/access/transam/xlogutils.c | 34 ++++++++++++++++++++++++++++++++++
 src/backend/storage/file/copydir.c     | 12 +++++++-----
 src/include/access/xlogutils.h         |  1 +
 3 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..75cdb882cd 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -522,6 +522,40 @@ XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 	return buffer;
 }
 
+/*
+ * XLogMakePGDirectory
+ *
+ * There is a possibility that WAL replay causes an error by creation of a
+ * directory under a directory removed before the previous crash. Issuing
+ * ERROR prevents the caller from continuing recovery.
+ *
+ * To prevent that case, this function issues WARNING instead of ERROR on
+ * error if consistency is not reached yet.
+ */
+int
+XLogMakePGDirectory(const char *directoryName)
+{
+	int ret;
+
+	ret = MakePGDirectory(directoryName);
+
+	if (ret != 0)
+	{
+		int elevel = ERROR;
+
+		/* Don't issue ERROR for this failure before reaching consistency. */
+		if (InRecovery && !reachedConsistency)
+			elevel = WARNING;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not create directory \"%s\": %m", directoryName)));
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * Struct actually returned by XLogFakeRelcacheEntry, though the declared
  * return type is Relation.
diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c
index 30f6200a86..0216270dd3 100644
--- a/src/backend/storage/file/copydir.c
+++ b/src/backend/storage/file/copydir.c
@@ -22,11 +22,11 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/xlogutils.h"
 #include "storage/copydir.h"
 #include "storage/fd.h"
 #include "miscadmin.h"
 #include "pgstat.h"
-
 /*
  * copydir: copy a directory
  *
@@ -41,10 +41,12 @@ copydir(char *fromdir, char *todir, bool recurse)
 	char		fromfile[MAXPGPATH * 2];
 	char		tofile[MAXPGPATH * 2];
 
-	if (MakePGDirectory(todir) != 0)
-		ereport(ERROR,
-				(errcode_for_file_access(),
-				 errmsg("could not create directory \"%s\": %m", todir)));
+	/*
+	 * We might have to skip copydir to continue recovery. See the function
+	 * for details.
+	 */
+	if (XLogMakePGDirectory(todir) != 0)
+		return;
 
 	xldir = AllocateDir(fromdir);
 
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0ab5ba62f5..46a7596315 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -43,6 +43,7 @@ extern XLogRedoAction XLogReadBufferForRedoExtended(XLogReaderState *record,
 
 extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 					   BlockNumber blkno, ReadBufferMode mode);
+extern int XLogMakePGDirectory(const char *directoryName);
 
 extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
 extern void FreeFakeRelcacheEntry(Relation fakerel);
-- 
2.16.3

From bc97e195f21af5d715d85424efc21fcbe8bb902c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Mon, 22 Apr 2019 20:59:15 +0900
Subject: [PATCH 4/5] Fix failure of standby startup caused by tablespace
 removal

When standby restarts after a crash after drop of a tablespace,
there's a possibility that recovery fails trying an object-creation in
already removed tablespace directory. Allow recovery to continue by
ignoring the error if not reaching consistency point.
---
 src/backend/access/transam/xlogutils.c | 34 ++++++++++++++++++++++++++++++++++
 src/backend/commands/tablespace.c      | 12 ++++++------
 src/backend/storage/file/copydir.c     | 12 +++++++-----
 src/include/access/xlogutils.h         |  1 +
 4 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..75cdb882cd 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -522,6 +522,40 @@ XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 	return buffer;
 }
 
+/*
+ * XLogMakePGDirectory
+ *
+ * There is a possibility that WAL replay causes an error by creation of a
+ * directory under a directory removed before the previous crash. Issuing
+ * ERROR prevents the caller from continuing recovery.
+ *
+ * To prevent that case, this function issues WARNING instead of ERROR on
+ * error if consistency is not reached yet.
+ */
+int
+XLogMakePGDirectory(const char *directoryName)
+{
+	int ret;
+
+	ret = MakePGDirectory(directoryName);
+
+	if (ret != 0)
+	{
+		int elevel = ERROR;
+
+		/* Don't issue ERROR for this failure before reaching consistency. */
+		if (InRecovery && !reachedConsistency)
+			elevel = WARNING;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not create directory \"%s\": %m", directoryName)));
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * Struct actually returned by XLogFakeRelcacheEntry, though the declared
  * return type is Relation.
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 66a70871e6..c9fb2af015 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -303,12 +303,6 @@ CreateTableSpace(CreateTableSpaceStmt *stmt)
 				(errcode(ERRCODE_INVALID_NAME),
 				 errmsg("tablespace location cannot contain single quotes")));
 
-	/* Reject tablespaces in the data directory. */
-	if (is_in_data_directory(location))
-		ereport(ERROR,
-				(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
-				 errmsg("tablespace location must not be inside the data directory")));
-
 	/*
 	 * Check that location isn't too long. Remember that we're going to append
 	 * 'PG_XXX/<dboid>/<relid>_<fork>.<nnn>'.  In the relative path case, we
@@ -323,6 +317,12 @@ CreateTableSpace(CreateTableSpaceStmt *stmt)
 				 errmsg("tablespace location \"%s\" is too long",
 						location)));
 
+	/* Reject tablespaces in the data directory. */
+	if (is_in_data_directory(location))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+				 errmsg("tablespace location must not be inside the data directory")));
+
 	/*
 	 * Disallow creation of tablespaces named "pg_xxx"; we reserve this
 	 * namespace for system purposes.
diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c
index 30f6200a86..0216270dd3 100644
--- a/src/backend/storage/file/copydir.c
+++ b/src/backend/storage/file/copydir.c
@@ -22,11 +22,11 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/xlogutils.h"
 #include "storage/copydir.h"
 #include "storage/fd.h"
 #include "miscadmin.h"
 #include "pgstat.h"
-
 /*
  * copydir: copy a directory
  *
@@ -41,10 +41,12 @@ copydir(char *fromdir, char *todir, bool recurse)
 	char		fromfile[MAXPGPATH * 2];
 	char		tofile[MAXPGPATH * 2];
 
-	if (MakePGDirectory(todir) != 0)
-		ereport(ERROR,
-				(errcode_for_file_access(),
-				 errmsg("could not create directory \"%s\": %m", todir)));
+	/*
+	 * We might have to skip copydir to continue recovery. See the function
+	 * for details.
+	 */
+	if (XLogMakePGDirectory(todir) != 0)
+		return;
 
 	xldir = AllocateDir(fromdir);
 
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0ab5ba62f5..46a7596315 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -43,6 +43,7 @@ extern XLogRedoAction XLogReadBufferForRedoExtended(XLogReaderState *record,
 
 extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
 					   BlockNumber blkno, ReadBufferMode mode);
+extern int XLogMakePGDirectory(const char *directoryName);
 
 extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
 extern void FreeFakeRelcacheEntry(Relation fakerel);
-- 
2.16.3

From 650221d14c1c34023aa66b1c22398eeae00dbcb8 Mon Sep 17 00:00:00 2001
From: Paul Guo <paulguo@gmail.com>
Date: Tue, 30 Apr 2019 13:30:49 +0800
Subject: [PATCH v2] skip copydir() if either src directory or dst directory is
 missing due to re-redoing create database but the tablespace is dropped.

---
 src/backend/access/rmgrdesc/dbasedesc.c | 14 ++++++----
 src/backend/commands/dbcommands.c       | 35 ++++++++++++++++++++++++-
 src/backend/commands/tablespace.c       | 28 +-------------------
 3 files changed, 44 insertions(+), 33 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index c7d60ce10d..35092ffb0e 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,21 +23,25 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char	   *dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec;
 
-		appendStringInfo(buf, "dir %u/%u",
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "dir %s", dbpath1);
+		pfree(dbpath1);
 	}
 }
 
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 9707afabd9..b7943529be 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -45,6 +45,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -2089,7 +2090,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    do_copydir = true;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2107,6 +2110,35 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that the tablespace was later dropped, but we are
+			 * re-redoing database create before that. In that case,
+			 * the directory are missing, we simply skip the copydir step.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				do_copydir = false;
+				ereport(WARNING,
+						(errmsg("directory \"%s\" for copydir() does not exists."
+								"It is possibly expected. Skip copydir().",
+								parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/* src directory is possibly missing also. See previous comment. */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			do_copydir = false;
+			ereport(WARNING,
+					(errmsg("source directory \"%s\" for copydir() does not exists."
+							"It is possibly expected. Skip copydir().",
+							src_path)));
+		}
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2119,7 +2151,8 @@ dbase_redo(XLogReaderState *record)
 		 *
 		 * We don't need to copy subdirectories
 		 */
-		copydir(src_path, dst_path, false);
+		if (do_copydir)
+			copydir(src_path, dst_path, false);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 8ec963f1cf..798c4586b8 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -154,8 +154,6 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -168,32 +166,8 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
-- 
2.17.2

From 968adb96f7440386e5ee70f66fbd7495068a09e6 Mon Sep 17 00:00:00 2001
From: Paul Guo <paulguo@gmail.com>
Date: Tue, 30 Apr 2019 13:30:49 +0800
Subject: [PATCH v3] skip copydir() if either src directory or dst directory is
 missing due to re-redoing create database but the tablespace is dropped.

Also correct dbase_desc() so that related xlog description is not misleading.

Add tap tests for the previous tablespace patch.

One of the test and related change in PostgresNode.pm are actually
from community (Kyotaro HORIGUCHI). I added another test and modified
PostgresNode.pm further to suport my added test.

This patch uses the log/forget mechanism to avoid bad ignoring.
---
 src/backend/access/rmgrdesc/dbasedesc.c   |  14 ++-
 src/backend/access/transam/xlog.c         |   4 +
 src/backend/commands/dbcommands.c         | 107 +++++++++++++++++++++-
 src/include/commands/dbcommands.h         |   2 +
 src/test/perl/PostgresNode.pm             |  13 ++-
 src/test/perl/RecursiveCopy.pm            |  33 ++++++-
 src/test/recovery/t/011_crash_recovery.pl |  99 +++++++++++++++++++-
 7 files changed, 259 insertions(+), 13 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index c7d60ce10d..35092ffb0e 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,21 +23,25 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char	   *dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec;
 
-		appendStringInfo(buf, "dir %u/%u",
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "dir %s", dbpath1);
+		pfree(dbpath1);
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e08320e829..5137a75efd 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -40,6 +40,7 @@
 #include "catalog/pg_control.h"
 #include "catalog/pg_database.h"
 #include "commands/tablespace.h"
+#include "commands/dbcommands.h"
 #include "common/controldata_utils.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -7842,6 +7843,9 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/* Check whether some missing directories are unexpected. */
+		CheckMissingDirs4DbaseRedo();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 15207bf75a..bc7ab3084e 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -45,6 +45,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -92,6 +93,68 @@ static void remove_dbtablespaces(Oid db_id);
 static bool check_db_file_conflict(Oid db_id);
 static int	errdetail_busy_db(int notherbackends, int npreparedxacts);
 
+static void log_missing_directory(char *dir);
+static void forget_missing_directory(char *dir);
+
+/*
+ * During XLOG replay, we may see either src directory or dst directory
+ * is missing during copying directory when creating database in dbase_redo()
+ * if the related tablespace was later dropped but we do re-redoing in
+ * recovery after abnormal shutdown. We do simply ignore copying in
+ * dbase_redo() but log those directories in memory and then sanity check
+ * the potential bug or bad user behaviors.
+ *
+ * We use List for simplicity since this should be ok for most cases - the
+ * list should be not long in usual case.
+ */
+static List *missing_dirs_dbase_redo = NIL;
+
+void
+CheckMissingDirs4DbaseRedo()
+{
+	ListCell *lc;
+
+	if (missing_dirs_dbase_redo == NIL)
+		return;
+
+	foreach(lc, missing_dirs_dbase_redo)
+	{
+		char *dir_entry = (char *) lfirst(lc);
+
+		elog(LOG, "Directory \"%s\" was missing during directory copying "
+			 "when replaying 'database create'", dir_entry);
+	}
+
+	elog(PANIC, "WAL replay was wrong due to previous missing directories");
+}
+
+static void
+log_missing_directory(char *dir)
+{
+	elog(DEBUG2, "Logging missing directory for dbase_redo(): \"%s\"", dir);
+	missing_dirs_dbase_redo = lappend(missing_dirs_dbase_redo, pstrdup(dir));
+}
+
+static void
+forget_missing_directory(char *dir)
+{
+	ListCell *prev, *lc;
+
+	prev = NULL;
+	foreach(lc, missing_dirs_dbase_redo)
+	{
+		char *dir_entry = (char *) lfirst(lc);
+
+		if (strcmp(dir_entry, dir) == 0)
+		{
+			missing_dirs_dbase_redo = list_delete_cell(missing_dirs_dbase_redo, lc, prev);
+			elog(DEBUG2, "forgetting missing directory for dbase_redo(): \"%s\"", dir);
+			return;
+		}
+
+		prev = lc;
+	}
+}
 
 /*
  * CREATE DATABASE
@@ -2089,7 +2152,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    do_copydir = true;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2107,6 +2172,43 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that the tablespace was previously dropped, but
+			 * we are re-redoing database create with that tablespace after
+			 * an abnormal shutdown (e.g. immediate shutdown). In that case,
+			 * the directory are missing, we simply skip the copydir step.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				do_copydir = false;
+				log_missing_directory(dst_path);
+				ereport(WARNING,
+						(errmsg("Skip creating database directory \"%s\". "
+								"The dest tablespace may have been removed "
+								"before abnormal shutdown. If the removal "
+								"is illegal after later checking we will panic.",
+								parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/* src directory is possibly missing during redo also. */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			do_copydir = false;
+			log_missing_directory(src_path);
+			ereport(WARNING,
+					(errmsg("Skip creating database directory based on "
+							"\"%s\". The src tablespace may have been "
+							"removed before abnormal shutdown. If the removal "
+							"is illegal after later checking we will panic.",
+							src_path)));
+
+		}
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2119,7 +2221,8 @@ dbase_redo(XLogReaderState *record)
 		 *
 		 * We don't need to copy subdirectories
 		 */
-		copydir(src_path, dst_path, false);
+		if (do_copydir)
+			copydir(src_path, dst_path, false);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -2162,6 +2265,8 @@ dbase_redo(XLogReaderState *record)
 					(errmsg("some useless files may be left behind in old database directory \"%s\"",
 							dst_path)));
 
+		forget_missing_directory(dst_path);
+
 		if (InHotStandby)
 		{
 			/*
diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h
index 28bf21153d..4893e0e289 100644
--- a/src/include/commands/dbcommands.h
+++ b/src/include/commands/dbcommands.h
@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern void CheckMissingDirs4DbaseRedo(void);
+
 extern Oid	createdb(ParseState *pstate, const CreatedbStmt *stmt);
 extern void dropdb(const char *dbname, bool missing_ok);
 extern ObjectAddress RenameDatabase(const char *oldname, const char *newname);
diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 8d5ad6bc16..e5b465dafd 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -551,13 +551,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..c912ce412d 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::real_dir($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 5dc52412ca..fad6a64d2a 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +66,100 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# Ensure that tablespace removal doesn't cause error while recoverying
+# the preceding create datbase or objects.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $tspDir_master = TestLib::tempdir;
+my $realTSDir_master = TestLib::real_dir($tspDir_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master'");
+
+my $tspDir_standby = TestLib::tempdir;
+my $realTSDir_standby = TestLib::real_dir($tspDir_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master=$realTSDir_standby");
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This leaves a CREATE DATBASE WAL record
+# that is to be applied to already-removed tablespace.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE db1 WITH TABLESPACE ts1;
+						  DROP DATABASE db1;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# Ensure that tablespace removal doesn't cause error while recoverying
+# the preceding create datbase or objects.
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+$tspDir_master = TestLib::tempdir;
+$realTSDir_master = TestLib::real_dir($tspDir_master);
+mkdir "$realTSDir_master/1";
+mkdir "$realTSDir_master/2";
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master/1'");
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts2 LOCATION '$realTSDir_master/2'");
+
+$tspDir_standby = TestLib::tempdir;
+$realTSDir_standby = TestLib::real_dir($tspDir_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master/1=$realTSDir_standby/1,$realTSDir_master/2=$realTSDir_standby/2");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown ...
+$node_master->safe_psql('postgres',
+						q[ALTER DATABASE db1 SET TABLESPACE ts2;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
-- 
2.17.2

From 7339a82089d2f363a55cb651d3651a2c01f2af7d Mon Sep 17 00:00:00 2001
From: Paul Guo <paulguo@gmail.com>
Date: Tue, 30 Apr 2019 13:30:49 +0800
Subject: [PATCH v4] skip copydir() if either src directory or dst directory is
 missing due to re-redoing create database but the tablespace is dropped.

Also correct dbase_desc() so that related xlog description is not misleading.

This patch uses the log/forget mechanism to avoid bad ignoring.

Kyotaro horiguchi added one test and did related change in PostgresNode.pm and
had a lot of discussion on this issue. I further added another test and
modified PostgresNode.pm again to support my new test.
---
 src/backend/access/rmgrdesc/dbasedesc.c   |  14 ++-
 src/backend/access/transam/xlog.c         |   4 +
 src/backend/commands/dbcommands.c         | 107 +++++++++++++++++++++-
 src/include/commands/dbcommands.h         |   2 +
 src/test/perl/PostgresNode.pm             |  13 ++-
 src/test/perl/RecursiveCopy.pm            |  33 ++++++-
 src/test/recovery/t/011_crash_recovery.pl |  99 +++++++++++++++++++-
 7 files changed, 259 insertions(+), 13 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index c7d60ce10d..35092ffb0e 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,21 +23,25 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char	   *dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec;
 
-		appendStringInfo(buf, "dir %u/%u",
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "dir %s", dbpath1);
+		pfree(dbpath1);
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b6c9353cbd..aa3e5c726c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -40,6 +40,7 @@
 #include "catalog/pg_control.h"
 #include "catalog/pg_database.h"
 #include "commands/tablespace.h"
+#include "commands/dbcommands.h"
 #include "common/controldata_utils.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -7855,6 +7856,9 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/* Check whether some missing directories are unexpected. */
+		CheckMissingDirs4DbaseRedo();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 863f89f19d..8ae804467e 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -45,6 +45,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -92,6 +93,68 @@ static void remove_dbtablespaces(Oid db_id);
 static bool check_db_file_conflict(Oid db_id);
 static int	errdetail_busy_db(int notherbackends, int npreparedxacts);
 
+static void log_missing_directory(char *dir);
+static void forget_missing_directory(char *dir);
+
+/*
+ * During XLOG replay, we may see either src directory or dst directory
+ * is missing during copying directory when creating database in dbase_redo()
+ * if the related tablespace was later dropped but we do re-redoing in
+ * recovery after abnormal shutdown. We do simply ignore copying in
+ * dbase_redo() but log those directories in memory and then sanity check
+ * the potential bug or bad user behaviors.
+ *
+ * We use List for simplicity since this should be ok for most cases - the
+ * list should be not long in usual case.
+ */
+static List *missing_dirs_dbase_redo = NIL;
+
+void
+CheckMissingDirs4DbaseRedo()
+{
+	ListCell *lc;
+
+	if (missing_dirs_dbase_redo == NIL)
+		return;
+
+	foreach(lc, missing_dirs_dbase_redo)
+	{
+		char *dir_entry = (char *) lfirst(lc);
+
+		elog(LOG, "Directory \"%s\" was missing during directory copying "
+			 "when replaying 'database create'", dir_entry);
+	}
+
+	elog(PANIC, "WAL replay was wrong due to previous missing directories");
+}
+
+static void
+log_missing_directory(char *dir)
+{
+	elog(DEBUG2, "Logging missing directory for dbase_redo(): \"%s\"", dir);
+	missing_dirs_dbase_redo = lappend(missing_dirs_dbase_redo, pstrdup(dir));
+}
+
+static void
+forget_missing_directory(char *dir)
+{
+	ListCell *prev, *lc;
+
+	prev = NULL;
+	foreach(lc, missing_dirs_dbase_redo)
+	{
+		char *dir_entry = (char *) lfirst(lc);
+
+		if (strcmp(dir_entry, dir) == 0)
+		{
+			missing_dirs_dbase_redo = list_delete_cell(missing_dirs_dbase_redo, lc, prev);
+			elog(DEBUG2, "forgetting missing directory for dbase_redo(): \"%s\"", dir);
+			return;
+		}
+
+		prev = lc;
+	}
+}
 
 /*
  * CREATE DATABASE
@@ -2108,7 +2171,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    do_copydir = true;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2126,6 +2191,43 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that the tablespace was previously dropped, but
+			 * we are re-redoing database create with that tablespace after
+			 * an abnormal shutdown (e.g. immediate shutdown). In that case,
+			 * the directory are missing, we simply skip the copydir step.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				do_copydir = false;
+				log_missing_directory(dst_path);
+				ereport(WARNING,
+						(errmsg("Skip creating database directory \"%s\". "
+								"The dest tablespace may have been removed "
+								"before abnormal shutdown. If the removal "
+								"is illegal after later checking we will panic.",
+								parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/* src directory is possibly missing during redo also. */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			do_copydir = false;
+			log_missing_directory(src_path);
+			ereport(WARNING,
+					(errmsg("Skip creating database directory based on "
+							"\"%s\". The src tablespace may have been "
+							"removed before abnormal shutdown. If the removal "
+							"is illegal after later checking we will panic.",
+							src_path)));
+
+		}
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2138,7 +2240,8 @@ dbase_redo(XLogReaderState *record)
 		 *
 		 * We don't need to copy subdirectories
 		 */
-		copydir(src_path, dst_path, false);
+		if (do_copydir)
+			copydir(src_path, dst_path, false);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -2181,6 +2284,8 @@ dbase_redo(XLogReaderState *record)
 					(errmsg("some useless files may be left behind in old database directory \"%s\"",
 							dst_path)));
 
+		forget_missing_directory(dst_path);
+
 		if (InHotStandby)
 		{
 			/*
diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h
index 28bf21153d..4893e0e289 100644
--- a/src/include/commands/dbcommands.h
+++ b/src/include/commands/dbcommands.h
@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern void CheckMissingDirs4DbaseRedo(void);
+
 extern Oid	createdb(ParseState *pstate, const CreatedbStmt *stmt);
 extern void dropdb(const char *dbname, bool missing_ok);
 extern ObjectAddress RenameDatabase(const char *oldname, const char *newname);
diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 6019f37f91..ba9b3f180f 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -542,13 +542,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..514ed90ae7 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::perl2host($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 5dc52412ca..a769236438 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +66,100 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# Ensure that tablespace removal doesn't cause error while recovering
+# the preceding create database with that tablespace.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $tspDir_master = TestLib::tempdir;
+my $realTSDir_master = TestLib::perl2host($tspDir_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master'");
+
+my $tspDir_standby = TestLib::tempdir;
+my $realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master=$realTSDir_standby");
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This leaves a CREATE DATBASE WAL record
+# that is to be applied to already-removed tablespace.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE db1 WITH TABLESPACE ts1;
+						  DROP DATABASE db1;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# Ensure that tablespace removal doesn't cause error while recovering the
+# preceding alter database set tablespace.
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+$tspDir_master = TestLib::tempdir;
+$realTSDir_master = TestLib::perl2host($tspDir_master);
+mkdir "$realTSDir_master/1";
+mkdir "$realTSDir_master/2";
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master/1'");
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts2 LOCATION '$realTSDir_master/2'");
+
+$tspDir_standby = TestLib::tempdir;
+$realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master/1=$realTSDir_standby/1,$realTSDir_master/2=$realTSDir_standby/2");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown ...
+$node_master->safe_psql('postgres',
+						q[ALTER DATABASE db1 SET TABLESPACE ts2;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
-- 
2.17.2

From 1b6c0c2c67cfcdedb3de93d9a048cf86e4ae04f6 Mon Sep 17 00:00:00 2001
From: Paul Guo <paulguo@pivotal.io>
Date: Tue, 30 Apr 2019 13:30:49 +0800
Subject: [PATCH v5] skip copydir() if either src directory or dst directory is
 missing due to re-redoing create database but the tablespace is dropped.

Also correct dbase_desc() so that related xlog description is not misleading.

This patch uses the log/forget mechanism to avoid bad ignoring.

Kyotaro horiguchi added one test and did related change in PostgresNode.pm and
had a lot of discussion on this issue. I further added another test and
modified PostgresNode.pm again to support my new test.
---
 src/backend/access/rmgrdesc/dbasedesc.c   |  14 +--
 src/backend/access/transam/xlog.c         |   4 +
 src/backend/commands/dbcommands.c         | 104 +++++++++++++++++++++-
 src/include/commands/dbcommands.h         |   2 +
 src/test/perl/PostgresNode.pm             |  13 ++-
 src/test/perl/RecursiveCopy.pm            |  33 ++++++-
 src/test/recovery/t/011_crash_recovery.pl |  99 +++++++++++++++++++-
 7 files changed, 256 insertions(+), 13 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index c7d60ce10d..35092ffb0e 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,21 +23,25 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char	   *dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec;
 
-		appendStringInfo(buf, "dir %u/%u",
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "dir %s", dbpath1);
+		pfree(dbpath1);
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e651a841bb..0c4928a7c7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -40,6 +40,7 @@
 #include "catalog/pg_control.h"
 #include "catalog/pg_database.h"
 #include "commands/tablespace.h"
+#include "commands/dbcommands.h"
 #include "common/controldata_utils.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -7858,6 +7859,9 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/* Check whether some missing directories are unexpected. */
+		CheckMissingDirs4DbaseRedo();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 95881a8550..bf85677bd4 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -45,6 +45,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -92,6 +93,65 @@ static void remove_dbtablespaces(Oid db_id);
 static bool check_db_file_conflict(Oid db_id);
 static int	errdetail_busy_db(int notherbackends, int npreparedxacts);
 
+static void log_missing_directory(char *dir);
+static void forget_missing_directory(char *dir);
+
+/*
+ * During XLOG replay, we may see either src directory or dst directory
+ * is missing during copying directory when creating database in dbase_redo()
+ * if the related tablespace was later dropped but we do re-redoing in
+ * recovery after abnormal shutdown. We do simply ignore copying in
+ * dbase_redo() but log those directories in memory and then sanity check
+ * the potential bug or bad user behaviors.
+ *
+ * We use List for simplicity since this should be ok for most cases - the
+ * list should be not long in usual case.
+ */
+static List *missing_dirs_dbase_redo = NIL;
+
+void
+CheckMissingDirs4DbaseRedo()
+{
+	ListCell *lc;
+
+	if (missing_dirs_dbase_redo == NIL)
+		return;
+
+	foreach(lc, missing_dirs_dbase_redo)
+	{
+		char *dir_entry = (char *) lfirst(lc);
+
+		elog(LOG, "Directory \"%s\" was missing during directory copying "
+			 "when replaying 'database create'", dir_entry);
+	}
+
+	elog(PANIC, "WAL replay was wrong due to previous missing directories");
+}
+
+static void
+log_missing_directory(char *dir)
+{
+	elog(DEBUG2, "Logging missing directory for dbase_redo(): \"%s\"", dir);
+	missing_dirs_dbase_redo = lappend(missing_dirs_dbase_redo, pstrdup(dir));
+}
+
+static void
+forget_missing_directory(char *dir)
+{
+	ListCell *lc;
+
+	foreach(lc, missing_dirs_dbase_redo)
+	{
+		char *dir_entry = (char *) lfirst(lc);
+
+		if (strcmp(dir_entry, dir) == 0)
+		{
+			missing_dirs_dbase_redo = list_delete_cell(missing_dirs_dbase_redo, lc);
+			elog(DEBUG2, "forgetting missing directory for dbase_redo(): \"%s\"", dir);
+			return;
+		}
+	}
+}
 
 /*
  * CREATE DATABASE
@@ -2129,7 +2189,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    do_copydir = true;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2147,6 +2209,43 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that the tablespace was previously dropped, but
+			 * we are re-redoing database create with that tablespace after
+			 * an abnormal shutdown (e.g. immediate shutdown). In that case,
+			 * the directory are missing, we simply skip the copydir step.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				do_copydir = false;
+				log_missing_directory(dst_path);
+				ereport(WARNING,
+						(errmsg("Skip creating database directory \"%s\". "
+								"The dest tablespace may have been removed "
+								"before abnormal shutdown. If the removal "
+								"is illegal after later checking we will panic.",
+								parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/* src directory is possibly missing during redo also. */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			do_copydir = false;
+			log_missing_directory(src_path);
+			ereport(WARNING,
+					(errmsg("Skip creating database directory based on "
+							"\"%s\". The src tablespace may have been "
+							"removed before abnormal shutdown. If the removal "
+							"is illegal after later checking we will panic.",
+							src_path)));
+
+		}
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2159,7 +2258,8 @@ dbase_redo(XLogReaderState *record)
 		 *
 		 * We don't need to copy subdirectories
 		 */
-		copydir(src_path, dst_path, false);
+		if (do_copydir)
+			copydir(src_path, dst_path, false);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -2202,6 +2302,8 @@ dbase_redo(XLogReaderState *record)
 					(errmsg("some useless files may be left behind in old database directory \"%s\"",
 							dst_path)));
 
+		forget_missing_directory(dst_path);
+
 		if (InHotStandby)
 		{
 			/*
diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h
index 154c8157ee..26e96b8957 100644
--- a/src/include/commands/dbcommands.h
+++ b/src/include/commands/dbcommands.h
@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern void CheckMissingDirs4DbaseRedo(void);
+
 extern Oid	createdb(ParseState *pstate, const CreatedbStmt *stmt);
 extern void dropdb(const char *dbname, bool missing_ok);
 extern ObjectAddress RenameDatabase(const char *oldname, const char *newname);
diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 270bd6c856..b5dc3a8918 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..514ed90ae7 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::perl2host($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..5ac7303dd6 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +66,100 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# Ensure that tablespace removal doesn't cause error while recovering
+# the preceding create database with that tablespace.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $tspDir_master = TestLib::tempdir;
+my $realTSDir_master = TestLib::perl2host($tspDir_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master'");
+
+my $tspDir_standby = TestLib::tempdir;
+my $realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master=$realTSDir_standby");
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This leaves a CREATE DATBASE WAL record
+# that is to be applied to already-removed tablespace.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE db1 WITH TABLESPACE ts1;
+						  DROP DATABASE db1;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# Ensure that tablespace removal doesn't cause error while recovering the
+# preceding alter database set tablespace.
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+$tspDir_master = TestLib::tempdir;
+$realTSDir_master = TestLib::perl2host($tspDir_master);
+mkdir "$realTSDir_master/1";
+mkdir "$realTSDir_master/2";
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master/1'");
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts2 LOCATION '$realTSDir_master/2'");
+
+$tspDir_standby = TestLib::tempdir;
+$realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master/1=$realTSDir_standby/1,$realTSDir_master/2=$realTSDir_standby/2");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown ...
+$node_master->safe_psql('postgres',
+						q[ALTER DATABASE db1 SET TABLESPACE ts2;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
-- 
2.17.2

From bd98ac1a2e98bc67238103b0f2764cf7fe0edc58 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Thu, 12 Sep 2019 17:17:29 +0530
Subject: [PATCH v1 1/2] Create restartpoint when replaying drop database

Drop database replay involves removing the database directory.  We do
not have a mechanism similar to invalid page detection for directories
during WAL replay.  If, due to a crash, WAL replay must resume from a
checkpoint, we should avoid replaying WAL records second time, that
precede the drop database.

Proposed by Paul Guo.
---
 src/backend/access/rmgrdesc/dbasedesc.c |  6 ++-
 src/backend/commands/dbcommands.c       | 84 +++++++++++++++++++++------------
 src/include/access/xlog.h               |  1 -
 src/include/commands/dbcommands_xlog.h  |  3 +-
 4 files changed, 61 insertions(+), 33 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index c7d60ce10d..b97bac2411 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -36,8 +36,10 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 	{
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec;
 
-		appendStringInfo(buf, "dir %u/%u",
-						 xlrec->tablespace_id, xlrec->db_id);
+		int i;
+		for (i = 0; i < xlrec->nspcids; i++)
+			appendStringInfo(buf, "\ndir %u/%u",
+							 xlrec->db_id, xlrec->tablespace_ids[i]);
 	}
 }
 
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 95881a8550..c0c8726698 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -1400,16 +1400,19 @@ movedb(const char *dbname, const char *tblspcname)
 	 * Record the filesystem change in XLOG
 	 */
 	{
-		xl_dbase_drop_rec xlrec;
+		size_t xlrec_size = sizeof(xl_dbase_drop_rec) + sizeof(Oid);
+		xl_dbase_drop_rec *xlrec = palloc(xlrec_size);
 
-		xlrec.db_id = db_id;
-		xlrec.tablespace_id = src_tblspcoid;
+		xlrec->db_id = db_id;
+		xlrec->nspcids = 1;
+		xlrec->tablespace_ids[0] = src_tblspcoid;
 
 		XLogBeginInsert();
-		XLogRegisterData((char *) &xlrec, sizeof(xl_dbase_drop_rec));
+		XLogRegisterData((char *) &xlrec, xlrec_size);
 
 		(void) XLogInsert(RM_DBASE_ID,
 						  XLOG_DBASE_DROP | XLR_SPECIAL_REL_UPDATE);
+		pfree(xlrec);
 	}
 
 	/* Now it's safe to release the database lock */
@@ -1914,6 +1917,8 @@ remove_dbtablespaces(Oid db_id)
 	Relation	rel;
 	TableScanDesc scan;
 	HeapTuple	tuple;
+	List *dstpaths = NIL;
+	List *spcoids = NIL;
 
 	rel = table_open(TableSpaceRelationId, AccessShareLock);
 	scan = table_beginscan_catalog(rel, 0, NULL);
@@ -1936,31 +1941,39 @@ remove_dbtablespaces(Oid db_id)
 			pfree(dstpath);
 			continue;
 		}
+		dstpaths = lappend(dstpaths, dstpath);
+		spcoids = lappend_oid(spcoids, dsttablespace);
+	}
+
+	table_endscan(scan);
+	table_close(rel, AccessShareLock);
+
+	size_t xlrec_size = (sizeof(xl_dbase_drop_rec) +
+						 sizeof(Oid)*list_length(spcoids));
+	xl_dbase_drop_rec *xlrec = palloc(xlrec_size);
+	xlrec->db_id = db_id;
+	xlrec->nspcids = list_length(spcoids);
 
-		if (!rmtree(dstpath, true))
+	int i=0;
+	const ListCell *cell1, *cell2;
+	forboth(cell1, dstpaths, cell2, spcoids)
+	{
+		char *path = lfirst(cell1);
+		if (!rmtree(path, true))
 			ereport(WARNING,
 					(errmsg("some useless files may be left behind in old database directory \"%s\"",
-							dstpath)));
-
+							path)));
+		pfree(path);
 		/* Record the filesystem change in XLOG */
-		{
-			xl_dbase_drop_rec xlrec;
-
-			xlrec.db_id = db_id;
-			xlrec.tablespace_id = dsttablespace;
-
-			XLogBeginInsert();
-			XLogRegisterData((char *) &xlrec, sizeof(xl_dbase_drop_rec));
-
-			(void) XLogInsert(RM_DBASE_ID,
-							  XLOG_DBASE_DROP | XLR_SPECIAL_REL_UPDATE);
-		}
-
-		pfree(dstpath);
+		xlrec->tablespace_ids[i++] = lfirst_oid(cell2);
 	}
 
-	table_endscan(scan);
-	table_close(rel, AccessShareLock);
+	XLogBeginInsert();
+	XLogRegisterData((char *) &xlrec, xlrec_size);
+	(void) XLogInsert(RM_DBASE_ID,
+					  XLOG_DBASE_DROP | XLR_SPECIAL_REL_UPDATE);
+
+	pfree(xlrec);
 }
 
 /*
@@ -2166,8 +2179,6 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) XLogRecGetData(record);
 		char	   *dst_path;
 
-		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
-
 		if (InHotStandby)
 		{
 			/*
@@ -2196,11 +2207,26 @@ dbase_redo(XLogReaderState *record)
 		/* Clean out the xlog relcache too */
 		XLogDropDatabase(xlrec->db_id);
 
+		/*
+		 * If we crash after removing directories, we should avoid replaying
+		 * WAL records prior to the current WAL record (drop database).
+		 * Creating a restartpoint ensures that recovery will start from at
+		 * least this point onwards in the event of a crash / immediate
+		 * shutdown.
+		 */
+		RequestCheckpoint(CHECKPOINT_IMMEDIATE | CHECKPOINT_FORCE |
+						  CHECKPOINT_WAIT);
+
 		/* And remove the physical files */
-		if (!rmtree(dst_path, true))
-			ereport(WARNING,
-					(errmsg("some useless files may be left behind in old database directory \"%s\"",
-							dst_path)));
+		int i;
+		for (i = 0; i < xlrec->nspcids; i++)
+		{
+			dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			if (!rmtree(dst_path, true))
+				ereport(WARNING,
+						(errmsg("some useless files may be left behind in old database directory \"%s\"",
+								dst_path)));
+		}
 
 		if (InHotStandby)
 		{
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index d519252aad..d0582a726b 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -221,7 +221,6 @@ extern bool XLOG_DEBUG;
 /* These indicate the cause of a checkpoint request */
 #define CHECKPOINT_CAUSE_XLOG	0x0080	/* XLOG consumption */
 #define CHECKPOINT_CAUSE_TIME	0x0100	/* Elapsed time */
-
 /*
  * Flag bits for the record being inserted, set using XLogSetRecordFlags().
  */
diff --git a/src/include/commands/dbcommands_xlog.h b/src/include/commands/dbcommands_xlog.h
index 46be8a615a..a1654f25e0 100644
--- a/src/include/commands/dbcommands_xlog.h
+++ b/src/include/commands/dbcommands_xlog.h
@@ -34,7 +34,8 @@ typedef struct xl_dbase_drop_rec
 {
 	/* Records dropping of a single subdirectory incl. contents */
 	Oid			db_id;
-	Oid			tablespace_id;
+	uint16		nspcids;
+	Oid			tablespace_ids[0];
 } xl_dbase_drop_rec;
 
 extern void dbase_redo(XLogReaderState *rptr);
-- 
2.14.3 (Apple Git-98)

From 7c311632576bc8f48ac43a9dd0fa07e86e6b47e6 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Thu, 12 Sep 2019 17:19:45 +0530
Subject: [PATCH v1 2/2] Test to validate replay of WAL records involving drop
 database

Authored by Paul Guo and Kyotaro Horiguchi.
---
 src/test/perl/PostgresNode.pm             | 13 +++-
 src/test/perl/RecursiveCopy.pm            | 33 +++++++++--
 src/test/recovery/t/011_crash_recovery.pl | 99 ++++++++++++++++++++++++++++++-
 3 files changed, 138 insertions(+), 7 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 270bd6c856..b5dc3a8918 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..514ed90ae7 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::perl2host($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..5ac7303dd6 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +66,100 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# Ensure that tablespace removal doesn't cause error while recovering
+# the preceding create database with that tablespace.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $tspDir_master = TestLib::tempdir;
+my $realTSDir_master = TestLib::perl2host($tspDir_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master'");
+
+my $tspDir_standby = TestLib::tempdir;
+my $realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master=$realTSDir_standby");
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This leaves a CREATE DATBASE WAL record
+# that is to be applied to already-removed tablespace.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE db1 WITH TABLESPACE ts1;
+						  DROP DATABASE db1;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# Ensure that tablespace removal doesn't cause error while recovering the
+# preceding alter database set tablespace.
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+$tspDir_master = TestLib::tempdir;
+$realTSDir_master = TestLib::perl2host($tspDir_master);
+mkdir "$realTSDir_master/1";
+mkdir "$realTSDir_master/2";
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master/1'");
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts2 LOCATION '$realTSDir_master/2'");
+
+$tspDir_standby = TestLib::tempdir;
+$realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master/1=$realTSDir_standby/1,$realTSDir_master/2=$realTSDir_standby/2");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown ...
+$node_master->safe_psql('postgres',
+						q[ALTER DATABASE db1 SET TABLESPACE ts2;
+						  DROP TABLESPACE ts1;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
-- 
2.14.3 (Apple Git-98)

From 26b34d171d2bb185a2d927b88a77a4b0cacb0c88 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Thu, 19 Sep 2019 17:14:55 +0530
Subject: [PATCH] Tests for replay of create database operation on standby

A couple of tests to demonstrate that standby fails to replay a create
database WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a drop
tablespace or drop database WAL record has been replayed in archive
recovery, before a crash.  And then the create database record happens
to be replayed again during crash recovery.  The failures indicate bugs
that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra, Anastasia, Kyotaro, Paul and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 150 +++++++++++++++++++++++++++++-
 1 file changed, 149 insertions(+), 1 deletion(-)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..1cea17c7d4 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,6 +6,7 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 if ($Config{osname} eq 'MSWin32')
 {
@@ -15,7 +16,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +67,150 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is hangled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    tablespace.
+#
+# 2. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 3. Create a datbase from another database as template then drop the
+#    template database.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master = TestLib::tempdir;
+$dropme_ts_master = TestLib::perl2host($dropme_ts_master);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts location '$dropme_ts_master';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby = TestLib::tempdir;
+$dropme_ts_standby = TestLib::perl2host($dropme_ts_standby);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = "$dropme_ts_master=$dropme_ts_standby," .
+  "$source_ts_master=$source_ts_standby," .
+  "$target_ts_master=$target_ts_standby";
+$node_master->backup($backup_name, tablespace_mappings => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATBASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  DROP DATABASE dropme_db;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP TABLESPACE source_ts;
+						  DROP TABLESPACE dropme_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master4');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $tspDir_master = TestLib::tempdir;
+my $realTSDir_master = TestLib::perl2host($tspDir_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$realTSDir_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $tspDir_standby = TestLib::tempdir;
+my $realTSDir_standby = TestLib::perl2host($tspDir_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$realTSDir_master=$realTSDir_standby");
+$node_standby = get_new_node('standby4');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# XXX For some reason, the tablespace mapping is not honored and the
+# standby ends up getting a different temp dir than what was specified
+# in the tablepsace mapping.  So get the tablespace directory by
+# querying standby.
+$realTSDir_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($realTSDir_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until_params(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		timeout => 5) == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3 (Apple Git-98)

From f676cd4c9d698b0f9e4f86d6251253ac6ddda36f Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v6 1/3] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro
---
 src/test/perl/RecursiveCopy.pm | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..514ed90ae7 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::perl2host($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
-- 
2.14.3 (Apple Git-98)

From 45f2eb5ddd2c819942cc299e97eb017cd04b5181 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v6 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra, Anastasia, Kyotaro, Paul and me.
---
 src/test/perl/PostgresNode.pm             |  31 +++++-
 src/test/recovery/t/011_crash_recovery.pl | 152 +++++++++++++++++++++++++++++-
 2 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 270bd6c856..959fa24bba 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
@@ -1550,9 +1559,21 @@ sub poll_query_until
 
 	$expected = 't' unless defined($expected);    # default value
 
+	$self->poll_query_until_params(
+		$dbname, $query,
+		expected => $expected, timeout => 180);
+}
+
+sub poll_query_until_params
+{
+	my ($self, $dbname, $query, %params) = @_;
+
+	$params{expected} = 't' unless defined($params{expected});
+	$params{timeout} = 180 unless defined($params{timeout});
+
 	my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -1562,7 +1583,7 @@ sub poll_query_until
 		chomp($stdout);
 		$stdout =~ s/\r//g if $TestLib::windows_os;
 
-		if ($stdout eq $expected)
+		if ($stdout eq $params{expected})
 		{
 			return 1;
 		}
@@ -1580,7 +1601,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..23397232b8 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,6 +6,7 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 if ($Config{osname} eq 'MSWin32')
 {
@@ -15,7 +16,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +67,152 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is hangled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    tablespace.
+#
+# 2. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 3. Create a datbase from another database as template then drop the
+#    template database.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master = TestLib::tempdir;
+$dropme_ts_master = TestLib::perl2host($dropme_ts_master);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts location '$dropme_ts_master';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby = TestLib::tempdir;
+$dropme_ts_standby = TestLib::perl2host($dropme_ts_standby);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = "$dropme_ts_master=$dropme_ts_standby," .
+  "$source_ts_master=$source_ts_standby," .
+  "$target_ts_master=$target_ts_standby";
+$node_master->backup($backup_name, tablespace_mappings => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATBASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  DROP DATABASE dropme_db1;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE dropme_db2;
+						  DROP TABLESPACE dropme_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$ts_master=$ts_standby");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until_params(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		timeout => 5) == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3 (Apple Git-98)

From ee35455705ac7ea81b5db6e090591971d277c5a8 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:43:25 +0530
Subject: [PATCH v6 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul.

Authored by Paul, Kyotaro and me.

Discussion: CAEET0ZGx9AvioViLf7nbR_8tH9-%3D27DN5xWJ2P9-ROH16e4JUA%40mail.gmail.com
---
 src/backend/access/rmgrdesc/dbasedesc.c |  14 ++--
 src/backend/access/transam/xlog.c       |   6 ++
 src/backend/access/transam/xlogutils.c  | 130 ++++++++++++++++++++++++++++++++
 src/backend/commands/dbcommands.c       |  53 +++++++++++++
 src/backend/commands/tablespace.c       |   3 +
 src/include/access/xlogutils.h          |   4 +
 src/include/commands/dbcommands.h       |   2 +
 7 files changed, 207 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index c7d60ce10d..35092ffb0e 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,21 +23,25 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char	   *dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
 		xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec;
 
-		appendStringInfo(buf, "dir %u/%u",
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "dir %s", dbpath1);
+		pfree(dbpath1);
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b7ff004234..749e8d5961 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7858,6 +7858,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1fc39333f1..e1384afe30 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -56,6 +56,136 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (reachedConsistency)
+		elog(PANIC, "cannot find directory %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+		elog(DEBUG2, "missing directory %s tablespace %d database %d already exists: %s",
+			 path, spcNode, dbNode, entry->path);
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		elog(DEBUG2, "logged missing dir %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) == NULL)
+		elog(DEBUG2, "dir %s tablespace %d database %d is not missing",
+			 path, spcNode, dbNode);
+	else
+		elog(DEBUG2, "forgot missing dir %s for tablespace %d database %d",
+			 path, spcNode, dbNode);
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 95881a8550..5388800660 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -45,6 +45,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -2129,7 +2130,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2147,6 +2150,54 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogLogMissingDir(xlrec->tablespace_id, InvalidOid, dst_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			XLogLogMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2202,6 +2253,8 @@ dbase_redo(XLogReaderState *record)
 					(errmsg("some useless files may be left behind in old database directory \"%s\"",
 							dst_path)));
 
+		XLogForgetMissingDir(xlrec->tablespace_id, xlrec->db_id, dst_path);
+
 		if (InHotStandby)
 		{
 			/*
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 84efb414d8..0d553c39c8 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1517,6 +1518,8 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		XLogForgetMissingDir(xlrec->ts_id, InvalidOid, "");
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..7034b691bd 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h
index 154c8157ee..26e96b8957 100644
--- a/src/include/commands/dbcommands.h
+++ b/src/include/commands/dbcommands.h
@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern void CheckMissingDirs4DbaseRedo(void);
+
 extern Oid	createdb(ParseState *pstate, const CreatedbStmt *stmt);
 extern void dropdb(const char *dbname, bool missing_ok);
 extern ObjectAddress RenameDatabase(const char *oldname, const char *newname);
-- 
2.14.3 (Apple Git-98)

From f676cd4c9d698b0f9e4f86d6251253ac6ddda36f Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v6 1/3] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro
---
 src/test/perl/RecursiveCopy.pm | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..514ed90ae7 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -22,6 +22,7 @@ use warnings;
 use Carp;
 use File::Basename;
 use File::Copy;
+use TestLib;
 
 =pod
 
@@ -97,14 +98,38 @@ sub _copypath_recurse
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# (note: this will fall through quietly if file is already gone)
+	if (-l $srcpath)
+	{
+		croak "Cannot operate on symlink \"$srcpath\""
+		  if ($srcpath !~ /\/(pg_tblspc\/[0-9]+)$/);
+
+		# We have mapped tablespaces. Copy them individually
+		my $linkname = $1;
+		my $tmpdir = TestLib::tempdir;
+		my $dstrealdir = TestLib::perl2host($tmpdir);
+		my $srcrealdir = readlink($srcpath);
+
+		opendir(my $dh, $srcrealdir);
+		while (readdir $dh)
+		{
+			next if (/^\.\.?$/);
+			my $spath = "$srcrealdir/$_";
+			my $dpath = "$dstrealdir/$_";
+
+			copypath($spath, $dpath);
+		}
+		closedir $dh;
+
+		symlink $dstrealdir, $destpath;
+		return 1;
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
-- 
2.14.3 (Apple Git-98)

From 45f2eb5ddd2c819942cc299e97eb017cd04b5181 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v6 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra, Anastasia, Kyotaro, Paul and me.
---
 src/test/perl/PostgresNode.pm             |  31 +++++-
 src/test/recovery/t/011_crash_recovery.pl | 152 +++++++++++++++++++++++++++++-
 2 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 270bd6c856..959fa24bba 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
@@ -1550,9 +1559,21 @@ sub poll_query_until
 
 	$expected = 't' unless defined($expected);    # default value
 
+	$self->poll_query_until_params(
+		$dbname, $query,
+		expected => $expected, timeout => 180);
+}
+
+sub poll_query_until_params
+{
+	my ($self, $dbname, $query, %params) = @_;
+
+	$params{expected} = 't' unless defined($params{expected});
+	$params{timeout} = 180 unless defined($params{timeout});
+
 	my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -1562,7 +1583,7 @@ sub poll_query_until
 		chomp($stdout);
 		$stdout =~ s/\r//g if $TestLib::windows_os;
 
-		if ($stdout eq $expected)
+		if ($stdout eq $params{expected})
 		{
 			return 1;
 		}
@@ -1580,7 +1601,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..23397232b8 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,6 +6,7 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 if ($Config{osname} eq 'MSWin32')
 {
@@ -15,7 +16,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +67,152 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is hangled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    tablespace.
+#
+# 2. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 3. Create a datbase from another database as template then drop the
+#    template database.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master = TestLib::tempdir;
+$dropme_ts_master = TestLib::perl2host($dropme_ts_master);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts location '$dropme_ts_master';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby = TestLib::tempdir;
+$dropme_ts_standby = TestLib::perl2host($dropme_ts_standby);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = "$dropme_ts_master=$dropme_ts_standby," .
+  "$source_ts_master=$source_ts_standby," .
+  "$target_ts_master=$target_ts_standby";
+$node_master->backup($backup_name, tablespace_mappings => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATBASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  DROP DATABASE dropme_db1;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE dropme_db2;
+						  DROP TABLESPACE dropme_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$ts_master=$ts_standby");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until_params(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		timeout => 5) == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3 (Apple Git-98)

commit c1aa5f67df052467dd6d67cdf5dbd5388ecefb1f
Author: Anastasia <a.lubennikova@postgrespro.ru>
Date:   Thu Dec 26 17:30:33 2019 +0300

    Subject: [PATCH v7 3/3] Fix replay of create database records on standby
    
    Crash recovery on standby may encounter missing directories when
    replaying create database WAL records.  Prior to this patch, the
    standby would fail to recover in such a case.  However, the
    directories could be legitimately missing.  Consider a sequence of WAL
    records as follows:
    
        CREATE DATABASE
        DROP DATABASE
        DROP TABLESPACE
    
    If, after replaying the last WAL record and removing the tablespace
    directory, the standby crashes and has to replay the create database
    record again, the crash recovery must be able to move on.
    
    This patch adds mechanism similar to invalid page hash table, to track
    missing directories during crash recovery.  If all the missing
    directory references are matched with corresponding drop records at
    the end of crash recovery, the standby can safely enter archive
    recovery.
    
    Bug identified by Paul.
    
    Authored by Paul, Kyotaro and Asim R P.

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index d08c575..4858ce6 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,14 +23,17 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char		*dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -39,8 +42,11 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfo(buf, "dir");
 		for (i = 0; i < xlrec->ntablespaces; i++)
-			appendStringInfo(buf, " %u/%u",
-							 xlrec->tablespace_ids[i], xlrec->db_id);
+		{
+			dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			appendStringInfo(buf,  "%s", dbpath1);
+			pfree(dbpath1);
+		}
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5658971..ea6661e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7890,6 +7890,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 14efbf3..1417707 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -56,6 +56,136 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (reachedConsistency)
+		elog(PANIC, "cannot find directory %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+		elog(DEBUG2, "missing directory %s tablespace %d database %d already exists: %s",
+			 path, spcNode, dbNode, entry->path);
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		elog(DEBUG2, "logged missing dir %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) == NULL)
+		elog(DEBUG2, "dir %s tablespace %d database %d is not missing",
+			 path, spcNode, dbNode);
+	else
+		elog(DEBUG2, "forgot missing dir %s for tablespace %d database %d",
+			 path, spcNode, dbNode);
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index da0e5d8..b7cbb88 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -46,6 +46,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -2185,7 +2186,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2203,6 +2206,54 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogLogMissingDir(xlrec->tablespace_id, InvalidOid, dst_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			XLogLogMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2260,6 +2311,9 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id, dst_path);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 570dcb2..e4f6aad 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1516,6 +1517,8 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		XLogForgetMissingDir(xlrec->ts_id, InvalidOid, "");
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0572b24..938a35f 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h
index d1e91a2..9d321e7 100644
--- a/src/include/commands/dbcommands.h
+++ b/src/include/commands/dbcommands.h
@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern void CheckMissingDirs4DbaseRedo(void);
+
 extern Oid	createdb(ParseState *pstate, const CreatedbStmt *stmt);
 extern void dropdb(const char *dbname, bool missing_ok, bool force);
 extern void DropDatabase(ParseState *pstate, DropdbStmt *stmt);

From a81e747f0bfa90af8021e2399e196e177a93f62c Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v8 1/3] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro
---
 src/test/perl/PostgresNode.pm  | 29 +++++++++++++++++++++++-
 src/test/perl/RecursiveCopy.pm | 40 ++++++++++++++++++++++++++++------
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 2e0cf4a2f3..3cae483ddb 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -593,6 +593,32 @@ sub backup_fs_cold
 	return;
 }
 
+sub _srcsymlink
+{
+	my ($srcpath, $destpath) = @_;
+
+	croak "Cannot operate on symlink \"$srcpath\""
+		if ($srcpath !~ qr{/(pg_tblspc/[0-9]+)$});
+
+	# We have mapped tablespaces. Copy them individually
+	my $tmpdir = TestLib::tempdir;
+	my $dstrealdir = TestLib::perl2host($tmpdir);
+	my $srcrealdir = readlink($srcpath);
+
+	opendir(my $dh, $srcrealdir);
+	while (readdir $dh)
+	{
+		next if (/^\.\.?$/);
+		my $spath = "$srcrealdir/$_";
+		my $dpath = "$dstrealdir/$_";
+		RecursiveCopy::copypath($spath, $dpath);
+	}
+	closedir $dh;
+
+	symlink $dstrealdir, $destpath;
+
+	return 1;
+}
 
 # Common sub of backup_fs_hot and backup_fs_cold
 sub _backup_fs
@@ -680,7 +706,8 @@ sub init_from_backup
 
 	my $data_path = $self->data_dir;
 	rmdir($data_path);
-	RecursiveCopy::copypath($backup_path, $data_path);
+	RecursiveCopy::copypath($backup_path, $data_path,
+							srcsymlinkfn => \&_srcsymlink);
 	chmod(0700, $data_path);
 
 	# Base configuration for this node
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..715edcdedd 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -66,6 +66,7 @@ sub copypath
 {
 	my ($base_src_dir, $base_dest_dir, %params) = @_;
 	my $filterfn;
+	my $srcsymlinkfn;
 
 	if (defined $params{filterfn})
 	{
@@ -80,31 +81,55 @@ sub copypath
 		$filterfn = sub { return 1; };
 	}
 
+	if (defined $params{srcsymlinkfn})
+	{
+		croak "if specified, srcsymlinkfn must be a subroutine reference"
+			unless defined(ref $params{srcsymlinkfn})
+			and (ref $params{srcsymlinkfn} eq 'CODE');
+
+		$srcsymlinkfn = $params{srcsymlinkfn};
+	}
+	else
+	{
+		$srcsymlinkfn = undef;
+	}
+
 	# Complain if original path is bogus, because _copypath_recurse won't.
 	croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir;
 
 	# Start recursive copy from current directory
-	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn);
+	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn, $srcsymlinkfn);
 }
 
 # Recursive private guts of copypath
 sub _copypath_recurse
 {
-	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_;
+	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn,
+		$srcsymlinkfn) = @_;
 	my $srcpath  = "$base_src_dir/$curr_path";
 	my $destpath = "$base_dest_dir/$curr_path";
 
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# If caller provided us with a callback, call it; otherwise we're out.
+	if (-l $srcpath)
+	{
+		if (defined $srcsymlinkfn)
+		{
+			return &$srcsymlinkfn($srcpath, $destpath);
+		}
+		else
+		{
+			croak "Cannot operate on symlink \"$srcpath\"";
+		}
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
@@ -137,7 +162,8 @@ sub _copypath_recurse
 		{
 			next if ($entry eq '.' or $entry eq '..');
 			_copypath_recurse($base_src_dir, $base_dest_dir,
-				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn)
+				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn,
+				$srcsymlinkfn)
 			  or die "copypath $srcpath/$entry -> $destpath/$entry failed";
 		}
 
-- 
2.20.1

From 32e2c106aee30202f5731a163a6e2f1a88a6d06b Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v8 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra, Anastasia, Kyotaro, Paul and me.
---
 src/test/perl/PostgresNode.pm             |  34 ++++-
 src/test/recovery/t/011_crash_recovery.pl | 152 +++++++++++++++++++++-
 2 files changed, 178 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 3cae483ddb..e6e7ea505d 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
@@ -1640,13 +1649,24 @@ Returns 1 if successful, 0 if timed out.
 
 sub poll_query_until
 {
-	my ($self, $dbname, $query, $expected) = @_;
+	my ($self, $dbname, $query, $params) = @_;
+	my $expected;
 
-	$expected = 't' unless defined($expected);    # default value
+	# Be backwards-compatible
+	if (defined $params and ref $params eq '')
+	{
+		$params = {
+			expected => $params,
+			timeout => 180
+		};
+	}
+
+	$params->{expected} = 't' unless defined($params->{expected});
+	$params->{timeout} = 180 unless defined($params->{timeout});
 
 	my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params->{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -1656,7 +1676,7 @@ sub poll_query_until
 		chomp($stdout);
 		$stdout =~ s/\r//g if $TestLib::windows_os;
 
-		if ($stdout eq $expected)
+		if ($stdout eq $params->{expected})
 		{
 			return 1;
 		}
@@ -1674,7 +1694,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params->{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..013d3d5b0c 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,6 +6,7 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 if ($Config{osname} eq 'MSWin32')
 {
@@ -15,7 +16,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +67,152 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is hangled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    tablespace.
+#
+# 2. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 3. Create a datbase from another database as template then drop the
+#    template database.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master = TestLib::tempdir;
+$dropme_ts_master = TestLib::perl2host($dropme_ts_master);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts location '$dropme_ts_master';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby = TestLib::tempdir;
+$dropme_ts_standby = TestLib::perl2host($dropme_ts_standby);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = "$dropme_ts_master=$dropme_ts_standby," .
+  "$source_ts_master=$source_ts_standby," .
+  "$target_ts_master=$target_ts_standby";
+$node_master->backup($backup_name, tablespace_mappings => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATBASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  DROP DATABASE dropme_db1;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE dropme_db2;
+						  DROP TABLESPACE dropme_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$ts_master=$ts_standby");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		timeout => 5) == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.20.1

From 47ee0330541f22cfd934c59cc1ae6df34b08eea6 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v8 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul.

Authored by Paul, Kyotaro and Asim R P.
---
 src/backend/access/rmgrdesc/dbasedesc.c |  16 ++-
 src/backend/access/transam/xlog.c       |   6 ++
 src/backend/access/transam/xlogutils.c  | 130 ++++++++++++++++++++++++
 src/backend/commands/dbcommands.c       |  54 ++++++++++
 src/backend/commands/tablespace.c       |   3 +
 src/include/access/xlogutils.h          |   4 +
 src/include/commands/dbcommands.h       |   2 +
 7 files changed, 210 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index 73d2a4ca34..f7117873d7 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,14 +23,17 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char		*dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -39,8 +42,11 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfo(buf, "dir");
 		for (i = 0; i < xlrec->ntablespaces; i++)
-			appendStringInfo(buf, " %u/%u",
-							 xlrec->tablespace_ids[i], xlrec->db_id);
+		{
+			dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			appendStringInfo(buf,  "%s", dbpath1);
+			pfree(dbpath1);
+		}
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7f4f784c0e..d97e48f369 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7890,6 +7890,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b55c383370..6c2dd5aba1 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -56,6 +56,136 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (reachedConsistency)
+		elog(PANIC, "cannot find directory %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+		elog(DEBUG2, "missing directory %s tablespace %d database %d already exists: %s",
+			 path, spcNode, dbNode, entry->path);
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		elog(DEBUG2, "logged missing dir %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) == NULL)
+		elog(DEBUG2, "dir %s tablespace %d database %d is not missing",
+			 path, spcNode, dbNode);
+	else
+		elog(DEBUG2, "forgot missing dir %s for tablespace %d database %d",
+			 path, spcNode, dbNode);
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 367c30adb0..6d6668e4f8 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -46,6 +46,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -2185,7 +2186,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2203,6 +2206,54 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogLogMissingDir(xlrec->tablespace_id, InvalidOid, dst_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			XLogLogMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2260,6 +2311,9 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id, dst_path);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 051478057f..33407dceeb 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1516,6 +1517,8 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		XLogForgetMissingDir(xlrec->ts_id, InvalidOid, "");
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..4106735006 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h
index f8f6d5ffd0..b71b400e70 100644
--- a/src/include/commands/dbcommands.h
+++ b/src/include/commands/dbcommands.h
@@ -19,6 +19,8 @@
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 
+extern void CheckMissingDirs4DbaseRedo(void);
+
 extern Oid	createdb(ParseState *pstate, const CreatedbStmt *stmt);
 extern void dropdb(const char *dbname, bool missing_ok, bool force);
 extern void DropDatabase(ParseState *pstate, DropdbStmt *stmt);
-- 
2.20.1

From fa8b7c964d170bbf7aeb1a7b0e94de28f651f388 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v9 1/3] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro
---
 src/test/perl/PostgresNode.pm  | 29 ++++++++++++++++++++++++++++-
 src/test/perl/RecursiveCopy.pm | 40 +++++++++++++++++++++++++++++++++-------
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 2e0cf4a2f3e..587a12ef405 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -593,6 +593,32 @@ sub backup_fs_cold
 	return;
 }
 
+sub _srcsymlink
+{
+	my ($srcpath, $destpath) = @_;
+
+	croak "Cannot operate on symlink \"$srcpath\""
+		if ($srcpath !~ qr{/(pg_tblspc/[0-9]+)$});
+
+	# We have mapped tablespaces. Copy them individually
+	my $tmpdir = TestLib::tempdir;
+	my $dstrealdir = TestLib::perl2host($tmpdir);
+	my $srcrealdir = readlink($srcpath);
+
+	opendir(my $dh, $srcrealdir);
+	while (my $entry = (readdir $dh))
+	{
+		next if ($entry eq '.' or $entry eq '..');
+		my $spath = "$srcrealdir/$entry";
+		my $dpath = "$dstrealdir/$entry";
+		RecursiveCopy::copypath($spath, $dpath);
+	}
+	closedir $dh;
+
+	symlink $dstrealdir, $destpath;
+
+	return 1;
+}
 
 # Common sub of backup_fs_hot and backup_fs_cold
 sub _backup_fs
@@ -680,7 +706,8 @@ sub init_from_backup
 
 	my $data_path = $self->data_dir;
 	rmdir($data_path);
-	RecursiveCopy::copypath($backup_path, $data_path);
+	RecursiveCopy::copypath($backup_path, $data_path,
+							srcsymlinkfn => \&_srcsymlink);
 	chmod(0700, $data_path);
 
 	# Base configuration for this node
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63b..715edcdedd0 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -66,6 +66,7 @@ sub copypath
 {
 	my ($base_src_dir, $base_dest_dir, %params) = @_;
 	my $filterfn;
+	my $srcsymlinkfn;
 
 	if (defined $params{filterfn})
 	{
@@ -80,31 +81,55 @@ sub copypath
 		$filterfn = sub { return 1; };
 	}
 
+	if (defined $params{srcsymlinkfn})
+	{
+		croak "if specified, srcsymlinkfn must be a subroutine reference"
+			unless defined(ref $params{srcsymlinkfn})
+			and (ref $params{srcsymlinkfn} eq 'CODE');
+
+		$srcsymlinkfn = $params{srcsymlinkfn};
+	}
+	else
+	{
+		$srcsymlinkfn = undef;
+	}
+
 	# Complain if original path is bogus, because _copypath_recurse won't.
 	croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir;
 
 	# Start recursive copy from current directory
-	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn);
+	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn, $srcsymlinkfn);
 }
 
 # Recursive private guts of copypath
 sub _copypath_recurse
 {
-	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_;
+	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn,
+		$srcsymlinkfn) = @_;
 	my $srcpath  = "$base_src_dir/$curr_path";
 	my $destpath = "$base_dest_dir/$curr_path";
 
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# If caller provided us with a callback, call it; otherwise we're out.
+	if (-l $srcpath)
+	{
+		if (defined $srcsymlinkfn)
+		{
+			return &$srcsymlinkfn($srcpath, $destpath);
+		}
+		else
+		{
+			croak "Cannot operate on symlink \"$srcpath\"";
+		}
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
@@ -137,7 +162,8 @@ sub _copypath_recurse
 		{
 			next if ($entry eq '.' or $entry eq '..');
 			_copypath_recurse($base_src_dir, $base_dest_dir,
-				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn)
+				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn,
+				$srcsymlinkfn)
 			  or die "copypath $srcpath/$entry -> $destpath/$entry failed";
 		}
 
-- 
2.14.3

From 57c4d4f31e501e5b25bb36919e666855fd2b0789 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v9 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/rmgrdesc/dbasedesc.c |  16 ++--
 src/backend/access/transam/xlog.c       |   6 ++
 src/backend/access/transam/xlogutils.c  | 130 ++++++++++++++++++++++++++++++++
 src/backend/commands/dbcommands.c       |  54 +++++++++++++
 src/backend/commands/tablespace.c       |   3 +
 src/include/access/xlogutils.h          |   4 +
 6 files changed, 208 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index 73d2a4ca34b..f7117873d73 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,14 +23,17 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char		*dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -39,8 +42,11 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfo(buf, "dir");
 		for (i = 0; i < xlrec->ntablespaces; i++)
-			appendStringInfo(buf, " %u/%u",
-							 xlrec->tablespace_ids[i], xlrec->db_id);
+		{
+			dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			appendStringInfo(buf,  "%s", dbpath1);
+			pfree(dbpath1);
+		}
 	}
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7f4f784c0eb..d97e48f369c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7890,6 +7890,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b55c3833703..6c2dd5aba1b 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -56,6 +56,136 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (reachedConsistency)
+		elog(PANIC, "cannot find directory %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+		elog(DEBUG2, "missing directory %s tablespace %d database %d already exists: %s",
+			 path, spcNode, dbNode, entry->path);
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		elog(DEBUG2, "logged missing dir %s tablespace %d database %d",
+			 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) == NULL)
+		elog(DEBUG2, "dir %s tablespace %d database %d is not missing",
+			 path, spcNode, dbNode);
+	else
+		elog(DEBUG2, "forgot missing dir %s for tablespace %d database %d",
+			 path, spcNode, dbNode);
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 367c30adb01..6d6668e4f88 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -46,6 +46,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -2185,7 +2186,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2203,6 +2206,54 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogLogMissingDir(xlrec->tablespace_id, InvalidOid, dst_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			XLogLogMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2260,6 +2311,9 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id, dst_path);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 051478057f6..33407dceeb2 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1516,6 +1517,8 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		XLogForgetMissingDir(xlrec->ts_id, InvalidOid, "");
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d96..41067350069 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.14.3

From f1c9874eb75a9a2979e56402b4a5a363a6f061bc Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v9 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/perl/PostgresNode.pm             |  34 +++++--
 src/test/recovery/t/011_crash_recovery.pl | 152 +++++++++++++++++++++++++++++-
 2 files changed, 178 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 587a12ef405..4eef8bb1985 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
@@ -1640,13 +1649,24 @@ Returns 1 if successful, 0 if timed out.
 
 sub poll_query_until
 {
-	my ($self, $dbname, $query, $expected) = @_;
+	my ($self, $dbname, $query, $params) = @_;
+	my $expected;
+
+	# Be backwards-compatible
+	if (defined $params and ref $params eq '')
+	{
+		$params = {
+			expected => $params,
+			timeout => 180
+		};
+	}
 
-	$expected = 't' unless defined($expected);    # default value
+	$params->{expected} = 't' unless defined($params->{expected});
+	$params->{timeout} = 180 unless defined($params->{timeout});
 
 	my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params->{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -1656,7 +1676,7 @@ sub poll_query_until
 		chomp($stdout);
 		$stdout =~ s/\r//g if $TestLib::windows_os;
 
-		if ($stdout eq $expected)
+		if ($stdout eq $params->{expected})
 		{
 			return 1;
 		}
@@ -1674,7 +1694,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params->{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb5..3690865e07b 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,6 +6,7 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 if ($Config{osname} eq 'MSWin32')
 {
@@ -15,7 +16,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +67,152 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is hangled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    tablespace.
+#
+# 2. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 3. Create a datbase from another database as template then drop the
+#    template database.
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master = TestLib::tempdir;
+$dropme_ts_master = TestLib::perl2host($dropme_ts_master);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts location '$dropme_ts_master';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby = TestLib::tempdir;
+$dropme_ts_standby = TestLib::perl2host($dropme_ts_standby);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = "$dropme_ts_master=$dropme_ts_standby," .
+  "$source_ts_master=$source_ts_standby," .
+  "$target_ts_master=$target_ts_standby";
+$node_master->backup($backup_name, tablespace_mappings => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATBASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  DROP DATABASE dropme_db1;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE dropme_db2;
+						  DROP TABLESPACE dropme_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$ts_master=$ts_standby");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3

From b29747de93fb70d3a7a6f843e232d3dec747451e Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v10 1/4] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro Horiguchi
---
 src/test/perl/PostgresNode.pm  | 29 ++++++++++++++++++++++++++++-
 src/test/perl/RecursiveCopy.pm | 40 +++++++++++++++++++++++++++++++++-------
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 1407359aef6..e1ddb36ff1d 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -593,6 +593,32 @@ sub backup_fs_cold
 	return;
 }
 
+sub _srcsymlink
+{
+	my ($srcpath, $destpath) = @_;
+
+	croak "Cannot operate on symlink \"$srcpath\""
+		if ($srcpath !~ qr{/(pg_tblspc/[0-9]+)$});
+
+	# We have mapped tablespaces. Copy them individually
+	my $tmpdir = TestLib::tempdir;
+	my $dstrealdir = TestLib::perl2host($tmpdir);
+	my $srcrealdir = readlink($srcpath);
+
+	opendir(my $dh, $srcrealdir);
+	while (my $entry = (readdir $dh))
+	{
+		next if ($entry eq '.' or $entry eq '..');
+		my $spath = "$srcrealdir/$entry";
+		my $dpath = "$dstrealdir/$entry";
+		RecursiveCopy::copypath($spath, $dpath);
+	}
+	closedir $dh;
+
+	symlink $dstrealdir, $destpath;
+
+	return 1;
+}
 
 # Common sub of backup_fs_hot and backup_fs_cold
 sub _backup_fs
@@ -684,7 +710,8 @@ sub init_from_backup
 
 	my $data_path = $self->data_dir;
 	rmdir($data_path);
-	RecursiveCopy::copypath($backup_path, $data_path);
+	RecursiveCopy::copypath($backup_path, $data_path,
+							srcsymlinkfn => \&_srcsymlink);
 	chmod(0700, $data_path);
 
 	# Base configuration for this node
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63b..715edcdedd0 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -66,6 +66,7 @@ sub copypath
 {
 	my ($base_src_dir, $base_dest_dir, %params) = @_;
 	my $filterfn;
+	my $srcsymlinkfn;
 
 	if (defined $params{filterfn})
 	{
@@ -80,31 +81,55 @@ sub copypath
 		$filterfn = sub { return 1; };
 	}
 
+	if (defined $params{srcsymlinkfn})
+	{
+		croak "if specified, srcsymlinkfn must be a subroutine reference"
+			unless defined(ref $params{srcsymlinkfn})
+			and (ref $params{srcsymlinkfn} eq 'CODE');
+
+		$srcsymlinkfn = $params{srcsymlinkfn};
+	}
+	else
+	{
+		$srcsymlinkfn = undef;
+	}
+
 	# Complain if original path is bogus, because _copypath_recurse won't.
 	croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir;
 
 	# Start recursive copy from current directory
-	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn);
+	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn, $srcsymlinkfn);
 }
 
 # Recursive private guts of copypath
 sub _copypath_recurse
 {
-	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_;
+	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn,
+		$srcsymlinkfn) = @_;
 	my $srcpath  = "$base_src_dir/$curr_path";
 	my $destpath = "$base_dest_dir/$curr_path";
 
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# If caller provided us with a callback, call it; otherwise we're out.
+	if (-l $srcpath)
+	{
+		if (defined $srcsymlinkfn)
+		{
+			return &$srcsymlinkfn($srcpath, $destpath);
+		}
+		else
+		{
+			croak "Cannot operate on symlink \"$srcpath\"";
+		}
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
@@ -137,7 +162,8 @@ sub _copypath_recurse
 		{
 			next if ($entry eq '.' or $entry eq '..');
 			_copypath_recurse($base_src_dir, $base_dest_dir,
-				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn)
+				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn,
+				$srcsymlinkfn)
 			  or die "copypath $srcpath/$entry -> $destpath/$entry failed";
 		}
 
-- 
2.14.3

From 00a3a6e028afeca2495e55634cba53f0c813b241 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v10 2/4] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/perl/PostgresNode.pm             |  34 +++++--
 src/test/recovery/t/011_crash_recovery.pl | 162 +++++++++++++++++++++++++++++-
 2 files changed, 188 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index e1ddb36ff1d..81538bbbebe 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -546,13 +546,22 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @rest = ();
+
+	if (defined $params{tablespace_mappings})
+	{
+		my @ts_mappings = split(/,/, $params{tablespace_mappings});
+		foreach my $elem (@ts_mappings) {
+			push(@rest, '--tablespace-mapping='.$elem);
+		}
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h',
-		$self->host, '-p', $self->port, '--no-sync');
+		$self->host, '-p', $self->port, '--no-sync', @rest);
 	print "# Backup finished\n";
 	return;
 }
@@ -1666,13 +1675,24 @@ Returns 1 if successful, 0 if timed out.
 
 sub poll_query_until
 {
-	my ($self, $dbname, $query, $expected) = @_;
+	my ($self, $dbname, $query, $params) = @_;
+	my $expected;
+
+	# Be backwards-compatible
+	if (defined $params and ref $params eq '')
+	{
+		$params = {
+			expected => $params,
+			timeout => 180
+		};
+	}
 
-	$expected = 't' unless defined($expected);    # default value
+	$params->{expected} = 't' unless defined($params->{expected});
+	$params->{timeout} = 180 unless defined($params->{timeout});
 
 	my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params->{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -1682,7 +1702,7 @@ sub poll_query_until
 		chomp($stdout);
 		$stdout =~ s/\r//g if $TestLib::windows_os;
 
-		if ($stdout eq $expected)
+		if ($stdout eq $params->{expected})
 		{
 			return 1;
 		}
@@ -1700,7 +1720,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params->{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index ca6e92b50df..55f96519fa0 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,6 +6,7 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 if ($Config{osname} eq 'MSWin32')
 {
@@ -15,7 +16,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +67,162 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is handled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    database.
+#
+# 2. Create a database against a user-defined tablespace then drop the
+#    database and the tablespace.
+#
+# 3. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 4. Create a database from another database as template then drop the
+#    template database.
+#
+#
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master1 = TestLib::tempdir;
+$dropme_ts_master1 = TestLib::perl2host($dropme_ts_master1);
+my $dropme_ts_master2 = TestLib::tempdir;
+$dropme_ts_master2 = TestLib::perl2host($dropme_ts_master2);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts1 location '$dropme_ts_master1';
+						   CREATE TABLESPACE dropme_ts2 location '$dropme_ts_master2';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby1 = TestLib::tempdir;
+$dropme_ts_standby1 = TestLib::perl2host($dropme_ts_standby1);
+my $dropme_ts_standby2 = TestLib::tempdir;
+$dropme_ts_standby2 = TestLib::perl2host($dropme_ts_standby2);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = "$dropme_ts_master1=$dropme_ts_standby1," .
+  "$dropme_ts_master2=$dropme_ts_standby2," .
+  "$source_ts_master=$source_ts_standby," .
+  "$target_ts_master=$target_ts_standby";
+$node_master->backup($backup_name, tablespace_mappings => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATBASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 tablespace_mappings =>
+					   "$ts_master=$ts_standby");
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3

From 26b385bdcc42fe25c933a17ed68d636f57271138 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v10 3/4] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 ++
 src/backend/access/transam/xlogutils.c | 155 +++++++++++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  53 +++++++++++
 src/backend/commands/tablespace.c      |   5 ++
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 223 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeba..af4dd19a71d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8032,6 +8032,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5b..bd98e42c895 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -59,6 +59,161 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (reachedConsistency)
+	{
+		if (dbNode == InvalidOid)
+			elog(PANIC, "cannot find directory %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(PANIC, "cannot find directory %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index f27c3fe8c1c..4a3adc7c6fc 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2185,7 +2185,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2203,6 +2205,54 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogLogMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)))
+		{
+			XLogLogMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2260,6 +2310,9 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 051478057f6..5fd36f93197 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1516,6 +1517,10 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9f..34eecfab791 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogLogMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.14.3

From 7a002cd379e13794edae53aa926898e33445475d Mon Sep 17 00:00:00 2001
From: Paul Guo <guopa@vmware.com>
Date: Mon, 6 Jul 2020 21:20:15 +0800
Subject: [PATCH v10 4/4] Fix database create/drop wal description.

Previously the description messages are wrong since the database path is not
simply tablespce_oid/database_oid. Now we call GetDatabasePath() to get the
correct database path.
---
 src/backend/access/rmgrdesc/dbasedesc.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index d82484b9db4..8312ef8bd36 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,14 +23,17 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char		*dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -39,8 +42,11 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfo(buf, "dir");
 		for (i = 0; i < xlrec->ntablespaces; i++)
-			appendStringInfo(buf, " %u/%u",
-							 xlrec->tablespace_ids[i], xlrec->db_id);
+		{
+			dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			appendStringInfo(buf,  " %s", dbpath1);
+			pfree(dbpath1);
+		}
 	}
 }
 
-- 
2.14.3

From 4075252c30fd5728913ef78b99f6c6cc70875ecd Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v11 1/4] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro Horiguchi
---
 src/test/perl/PostgresNode.pm  | 29 ++++++++++++++++++++++++++-
 src/test/perl/RecursiveCopy.pm | 45 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 66 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 1e96357d7e..e27120e975 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -607,6 +607,32 @@ sub backup_fs_cold
 	return;
 }
 
+sub _srcsymlink
+{
+	my ($srcpath, $destpath) = @_;
+
+	croak "Cannot operate on symlink \"$srcpath\""
+		if ($srcpath !~ qr{/(pg_tblspc/[0-9]+)$});
+
+	# We have mapped tablespaces. Copy them individually
+	my $tmpdir = TestLib::tempdir;
+	my $dstrealdir = TestLib::perl2host($tmpdir);
+	my $srcrealdir = readlink($srcpath);
+
+	opendir(my $dh, $srcrealdir);
+	while (my $entry = (readdir $dh))
+	{
+		next if ($entry eq '.' or $entry eq '..');
+		my $spath = "$srcrealdir/$entry";
+		my $dpath = "$dstrealdir/$entry";
+		RecursiveCopy::copypath($spath, $dpath);
+	}
+	closedir $dh;
+
+	symlink $dstrealdir, $destpath;
+
+	return 1;
+}
 
 # Common sub of backup_fs_hot and backup_fs_cold
 sub _backup_fs
@@ -715,7 +741,8 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		RecursiveCopy::copypath($backup_path, $data_path);
+		RecursiveCopy::copypath($backup_path, $data_path,
+								srcsymlinkfn => \&_srcsymlink);
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index baf5d0ac63..aaa3ce8a1b 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -47,6 +47,11 @@ This subroutine will be called for each entry in the source directory with its
 relative path as only parameter; if the subroutine returns true the entry is
 copied, otherwise the file is skipped.
 
+If the B<srcsymlinkfn> parameter is given, it must be a subroutine reference.
+This subroutine will be called when the source directory is a symlink. It
+determines the mechanism that copies files from the source directory to the
+destination directory.
+
 On failure the target directory may be in some incomplete state; no cleanup is
 attempted.
 
@@ -66,6 +71,7 @@ sub copypath
 {
 	my ($base_src_dir, $base_dest_dir, %params) = @_;
 	my $filterfn;
+	my $srcsymlinkfn;
 
 	if (defined $params{filterfn})
 	{
@@ -80,31 +86,55 @@ sub copypath
 		$filterfn = sub { return 1; };
 	}
 
+	if (defined $params{srcsymlinkfn})
+	{
+		croak "if specified, srcsymlinkfn must be a subroutine reference"
+			unless defined(ref $params{srcsymlinkfn})
+			and (ref $params{srcsymlinkfn} eq 'CODE');
+
+		$srcsymlinkfn = $params{srcsymlinkfn};
+	}
+	else
+	{
+		$srcsymlinkfn = undef;
+	}
+
 	# Complain if original path is bogus, because _copypath_recurse won't.
 	croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir;
 
 	# Start recursive copy from current directory
-	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn);
+	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn, $srcsymlinkfn);
 }
 
 # Recursive private guts of copypath
 sub _copypath_recurse
 {
-	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_;
+	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn,
+		$srcsymlinkfn) = @_;
 	my $srcpath  = "$base_src_dir/$curr_path";
 	my $destpath = "$base_dest_dir/$curr_path";
 
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# If caller provided us with a callback, call it; otherwise we're out.
+	if (-l $srcpath)
+	{
+		if (defined $srcsymlinkfn)
+		{
+			return &$srcsymlinkfn($srcpath, $destpath);
+		}
+		else
+		{
+			croak "Cannot operate on symlink \"$srcpath\"";
+		}
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
@@ -137,7 +167,8 @@ sub _copypath_recurse
 		{
 			next if ($entry eq '.' or $entry eq '..');
 			_copypath_recurse($base_src_dir, $base_dest_dir,
-				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn)
+				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn,
+				$srcsymlinkfn)
 			  or die "copypath $srcpath/$entry -> $destpath/$entry failed";
 		}
 
-- 
2.14.3

From 154497b47de2de423b1ecf420d45abf2af0e1139 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v11 2/4] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/perl/PostgresNode.pm             |  21 +++-
 src/test/recovery/t/011_crash_recovery.pl | 162 +++++++++++++++++++++++++++++-
 2 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index e27120e975..d4e17644b4 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -1888,15 +1888,26 @@ Returns 1 if successful, 0 if timed out.
 
 sub poll_query_until
 {
-	my ($self, $dbname, $query, $expected) = @_;
+	my ($self, $dbname, $query, $params) = @_;
+	my $expected;
+
+	# Be backwards-compatible
+	if (defined $params and ref $params eq '')
+	{
+		$params = {
+			expected => $params,
+			timeout => 180
+		};
+	}
 
 	local %ENV = $self->_get_env();
 
-	$expected = 't' unless defined($expected);    # default value
+	$params->{expected} = 't' unless defined($params->{expected});
+	$params->{timeout} = 180 unless defined($params->{timeout});
 
 	my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params->{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -1906,7 +1917,7 @@ sub poll_query_until
 		$stdout =~ s/\r\n/\n/g if $Config{osname} eq 'msys';
 		chomp($stdout);
 
-		if ($stdout eq $expected)
+		if ($stdout eq $params->{expected})
 		{
 			return 1;
 		}
@@ -1924,7 +1935,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params->{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 10cd98f70a..faae3fb5e7 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -6,9 +6,10 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 
-plan tests => 3;
+plan tests => 5;
 
 my $node = get_new_node('primary');
 $node->init(allows_streaming => 1);
@@ -59,3 +60,162 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 
 $stdin .= "\\q\n";
 $tx->finish; # wait for psql to quit gracefully
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is handled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    database.
+#
+# 2. Create a database against a user-defined tablespace then drop the
+#    database and the tablespace.
+#
+# 3. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 4. Create a database from another database as template then drop the
+#    template database.
+#
+#
+
+my $node_master = get_new_node('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master1 = TestLib::tempdir;
+$dropme_ts_master1 = TestLib::perl2host($dropme_ts_master1);
+my $dropme_ts_master2 = TestLib::tempdir;
+$dropme_ts_master2 = TestLib::perl2host($dropme_ts_master2);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts1 location '$dropme_ts_master1';
+						   CREATE TABLESPACE dropme_ts2 location '$dropme_ts_master2';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby1 = TestLib::tempdir;
+$dropme_ts_standby1 = TestLib::perl2host($dropme_ts_standby1);
+my $dropme_ts_standby2 = TestLib::tempdir;
+$dropme_ts_standby2 = TestLib::perl2host($dropme_ts_standby2);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = [ "--tablespace-mapping=$dropme_ts_master1=$dropme_ts_standby1",
+  "--tablespace-mapping=$dropme_ts_master2=$dropme_ts_standby2",
+  "--tablespace-mapping=$source_ts_master=$source_ts_standby",
+  "--tablespace-mapping=$target_ts_master=$target_ts_standby" ];
+$node_master->backup($backup_name, backup_options => $ts_mapping);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = get_new_node('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 backup_options =>
+					   [ "--tablespace-mapping=$ts_master=$ts_standby" ]);
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3

From 16bf763dda617d6aaf454f8aff3c4d08e09b198b Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v11 3/4] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 ++
 src/backend/access/transam/xlogutils.c | 145 +++++++++++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  55 +++++++++++++
 src/backend/commands/tablespace.c      |   6 ++
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 216 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6f8810e149..b525679e8b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8175,6 +8175,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..8341b7a242 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -59,6 +59,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 2b159b60eb..7bd6d4efd9 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2185,7 +2185,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2203,6 +2205,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2260,6 +2311,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 69ea155d50..15eaa757cc 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1533,6 +1534,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..6561d9cebe 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.14.3

From 1939f82fc67d218b6990b8a694709f0a7cb59430 Mon Sep 17 00:00:00 2001
From: Paul Guo <guopa@vmware.com>
Date: Mon, 6 Jul 2020 21:20:15 +0800
Subject: [PATCH v11 4/4] Fix database create/drop wal description.

Previously the description messages are wrong since the database path is not
simply tablespce_oid/database_oid. Now we call GetDatabasePath() to get the
correct database path.

Authored by Paul Guo
---
 src/backend/access/rmgrdesc/dbasedesc.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index 26609845aa..873737161c 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,14 +23,17 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char		*dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -39,8 +42,11 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfoString(buf, "dir");
 		for (i = 0; i < xlrec->ntablespaces; i++)
-			appendStringInfo(buf, " %u/%u",
-							 xlrec->tablespace_ids[i], xlrec->db_id);
+		{
+			dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			appendStringInfo(buf,  " %s", dbpath1);
+			pfree(dbpath1);
+		}
 	}
 }
 
-- 
2.14.3

From 7c03826e7fb9872ea293f95d8270b3e64433949a Mon Sep 17 00:00:00 2001
From: Paul Guo <guopa@vmware.com>
Date: Mon, 6 Jul 2020 21:20:15 +0800
Subject: [PATCH v12 4/4] Fix database create/drop wal description.

Previously the description messages are wrong since the database path is not
simply tablespce_oid/database_oid. Now we call GetDatabasePath() to get the
correct database path.

Authored by Paul Guo
---
 src/backend/access/rmgrdesc/dbasedesc.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c
index 26609845aa..873737161c 100644
--- a/src/backend/access/rmgrdesc/dbasedesc.c
+++ b/src/backend/access/rmgrdesc/dbasedesc.c
@@ -23,14 +23,17 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 {
 	char	   *rec = XLogRecGetData(record);
 	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	char		*dbpath1, *dbpath2;
 
 	if (info == XLOG_DBASE_CREATE)
 	{
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) rec;
 
-		appendStringInfo(buf, "copy dir %u/%u to %u/%u",
-						 xlrec->src_tablespace_id, xlrec->src_db_id,
-						 xlrec->tablespace_id, xlrec->db_id);
+		dbpath1 = GetDatabasePath(xlrec->src_db_id,  xlrec->src_tablespace_id);
+		dbpath2 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
+		appendStringInfo(buf, "copy dir %s to %s", dbpath1, dbpath2);
+		pfree(dbpath2);
+		pfree(dbpath1);
 	}
 	else if (info == XLOG_DBASE_DROP)
 	{
@@ -39,8 +42,11 @@ dbase_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfoString(buf, "dir");
 		for (i = 0; i < xlrec->ntablespaces; i++)
-			appendStringInfo(buf, " %u/%u",
-							 xlrec->tablespace_ids[i], xlrec->db_id);
+		{
+			dbpath1 = GetDatabasePath(xlrec->db_id, xlrec->tablespace_ids[i]);
+			appendStringInfo(buf,  " %s", dbpath1);
+			pfree(dbpath1);
+		}
 	}
 }
 
-- 
2.14.3

From 6096936d13526f121db4d5f684053cdfb4881ebe Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:34:19 +0530
Subject: [PATCH v12 2/4] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/perl/PostgresNode.pm             |  21 +++-
 src/test/recovery/t/011_crash_recovery.pl | 162 +++++++++++++++++++++++++++++-
 2 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index b71e98f254..86e5b2f7be 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -2183,18 +2183,29 @@ Returns 1 if successful, 0 if timed out.
 
 sub poll_query_until
 {
-	my ($self, $dbname, $query, $expected) = @_;
+	my ($self, $dbname, $query, $params) = @_;
+	my $expected;
+
+	# Be backwards-compatible
+	if (defined $params and ref $params eq '')
+	{
+		$params = {
+			expected => $params,
+			timeout => 180
+		};
+	}
 
 	local %ENV = $self->_get_env();
 
-	$expected = 't' unless defined($expected);    # default value
+	$params->{expected} = 't' unless defined($params->{expected});
+	$params->{timeout} = 180 unless defined($params->{timeout});
 
 	my $cmd = [
 		$self->installed_command('psql'), '-XAt',
 		'-d',                             $self->connstr($dbname)
 	];
 	my ($stdout, $stderr);
-	my $max_attempts = 180 * 10;
+	my $max_attempts = $params->{timeout} * 10;
 	my $attempts     = 0;
 
 	while ($attempts < $max_attempts)
@@ -2207,7 +2218,7 @@ sub poll_query_until
 		$stderr =~ s/\r\n/\n/g if $Config{osname} eq 'msys';
 		chomp($stderr);
 
-		if ($stdout eq $expected && $stderr eq '')
+		if ($stdout eq $params->{expected} && $stderr eq '')
 		{
 			return 1;
 		}
@@ -2223,7 +2234,7 @@ sub poll_query_until
 	diag qq(poll_query_until timed out executing this query:
 $query
 expecting this output:
-$expected
+$params->{expected}
 last actual query output:
 $stdout
 with stderr:
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 72fc603e6d..fec330e808 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -9,9 +9,10 @@ use warnings;
 use PostgresNode;
 use TestLib;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 
-plan tests => 3;
+plan tests => 5;
 
 my $node = PostgresNode->new('primary');
 $node->init(allows_streaming => 1);
@@ -62,3 +63,162 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is handled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    database.
+#
+# 2. Create a database against a user-defined tablespace then drop the
+#    database and the tablespace.
+#
+# 3. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 4. Create a database from another database as template then drop the
+#    template database.
+#
+#
+
+my $node_master = PostgresNode->new('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master1 = TestLib::tempdir;
+$dropme_ts_master1 = TestLib::perl2host($dropme_ts_master1);
+my $dropme_ts_master2 = TestLib::tempdir;
+$dropme_ts_master2 = TestLib::perl2host($dropme_ts_master2);
+my $source_ts_master = TestLib::tempdir;
+$source_ts_master = TestLib::perl2host($source_ts_master);
+my $target_ts_master = TestLib::tempdir;
+$target_ts_master = TestLib::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts1 location '$dropme_ts_master1';
+						   CREATE TABLESPACE dropme_ts2 location '$dropme_ts_master2';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby1 = TestLib::tempdir;
+$dropme_ts_standby1 = TestLib::perl2host($dropme_ts_standby1);
+my $dropme_ts_standby2 = TestLib::tempdir;
+$dropme_ts_standby2 = TestLib::perl2host($dropme_ts_standby2);
+my $source_ts_standby = TestLib::tempdir;
+$source_ts_standby = TestLib::perl2host($source_ts_standby);
+my $target_ts_standby = TestLib::tempdir;
+$target_ts_standby = TestLib::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = [ "--tablespace-mapping=$dropme_ts_master1=$dropme_ts_standby1",
+  "--tablespace-mapping=$dropme_ts_master2=$dropme_ts_standby2",
+  "--tablespace-mapping=$source_ts_master=$source_ts_standby",
+  "--tablespace-mapping=$target_ts_master=$target_ts_standby" ];
+$node_master->backup($backup_name, backup_options => $ts_mapping);
+
+my $node_standby = PostgresNode->new('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = PostgresNode->new('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = TestLib::tempdir;
+$ts_master = TestLib::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = TestLib::tempdir("standby");
+$ts_standby = TestLib::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 backup_options =>
+					   [ "--tablespace-mapping=$ts_master=$ts_standby" ]);
+$node_standby = PostgresNode->new('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.14.3

From 5cd04cebf8361cfd7941063cab055a2608ff546f Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v12 1/4] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.

Authored by Kyotaro Horiguchi
---
 src/test/perl/PostgresNode.pm  | 29 ++++++++++++++++++++++++++-
 src/test/perl/RecursiveCopy.pm | 45 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 66 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 8158ea5b2f..b71e98f254 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -634,6 +634,32 @@ sub backup_fs_cold
 	return;
 }
 
+sub _srcsymlink
+{
+	my ($srcpath, $destpath) = @_;
+
+	croak "Cannot operate on symlink \"$srcpath\""
+		if ($srcpath !~ qr{/(pg_tblspc/[0-9]+)$});
+
+	# We have mapped tablespaces. Copy them individually
+	my $tmpdir = TestLib::tempdir;
+	my $dstrealdir = TestLib::perl2host($tmpdir);
+	my $srcrealdir = readlink($srcpath);
+
+	opendir(my $dh, $srcrealdir);
+	while (my $entry = (readdir $dh))
+	{
+		next if ($entry eq '.' or $entry eq '..');
+		my $spath = "$srcrealdir/$entry";
+		my $dpath = "$dstrealdir/$entry";
+		RecursiveCopy::copypath($spath, $dpath);
+	}
+	closedir $dh;
+
+	symlink $dstrealdir, $destpath;
+
+	return 1;
+}
 
 # Common sub of backup_fs_hot and backup_fs_cold
 sub _backup_fs
@@ -743,7 +769,8 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		RecursiveCopy::copypath($backup_path, $data_path);
+		RecursiveCopy::copypath($backup_path, $data_path,
+								srcsymlinkfn => \&_srcsymlink);
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm
index 8a9cc722b5..cbd7a874d7 100644
--- a/src/test/perl/RecursiveCopy.pm
+++ b/src/test/perl/RecursiveCopy.pm
@@ -49,6 +49,11 @@ This subroutine will be called for each entry in the source directory with its
 relative path as only parameter; if the subroutine returns true the entry is
 copied, otherwise the file is skipped.
 
+If the B<srcsymlinkfn> parameter is given, it must be a subroutine reference.
+This subroutine will be called when the source directory is a symlink. It
+determines the mechanism that copies files from the source directory to the
+destination directory.
+
 On failure the target directory may be in some incomplete state; no cleanup is
 attempted.
 
@@ -68,6 +73,7 @@ sub copypath
 {
 	my ($base_src_dir, $base_dest_dir, %params) = @_;
 	my $filterfn;
+	my $srcsymlinkfn;
 
 	if (defined $params{filterfn})
 	{
@@ -82,31 +88,55 @@ sub copypath
 		$filterfn = sub { return 1; };
 	}
 
+	if (defined $params{srcsymlinkfn})
+	{
+		croak "if specified, srcsymlinkfn must be a subroutine reference"
+			unless defined(ref $params{srcsymlinkfn})
+			and (ref $params{srcsymlinkfn} eq 'CODE');
+
+		$srcsymlinkfn = $params{srcsymlinkfn};
+	}
+	else
+	{
+		$srcsymlinkfn = undef;
+	}
+
 	# Complain if original path is bogus, because _copypath_recurse won't.
 	croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir;
 
 	# Start recursive copy from current directory
-	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn);
+	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn, $srcsymlinkfn);
 }
 
 # Recursive private guts of copypath
 sub _copypath_recurse
 {
-	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_;
+	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn,
+		$srcsymlinkfn) = @_;
 	my $srcpath  = "$base_src_dir/$curr_path";
 	my $destpath = "$base_dest_dir/$curr_path";
 
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# If caller provided us with a callback, call it; otherwise we're out.
+	if (-l $srcpath)
+	{
+		if (defined $srcsymlinkfn)
+		{
+			return &$srcsymlinkfn($srcpath, $destpath);
+		}
+		else
+		{
+			croak "Cannot operate on symlink \"$srcpath\"";
+		}
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
@@ -139,7 +169,8 @@ sub _copypath_recurse
 		{
 			next if ($entry eq '.' or $entry eq '..');
 			_copypath_recurse($base_src_dir, $base_dest_dir,
-				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn)
+				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn,
+				$srcsymlinkfn)
 			  or die "copypath $srcpath/$entry -> $destpath/$entry failed";
 		}
 
-- 
2.14.3

From 42eda96adda2815ac91f4323964d8cbf7863b23c Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v12 3/4] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 ++
 src/backend/access/transam/xlogutils.c | 145 +++++++++++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  55 +++++++++++++
 src/backend/commands/tablespace.c      |   5 ++
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 215 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8b39a2fdaa..d48716b506 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8172,6 +8172,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b1702bc6be..13d0f1c101 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 029fab48df..0f483edb71 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2143,7 +2143,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2161,6 +2163,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2218,6 +2269,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index a54239a8b3..2a4d14e29f 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -1531,6 +1531,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index a5cb3d322c..2ee3bd378d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.14.3

From aa6b0b94e42550f23c4cecfa23ea1face7d71bc6 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Mon, 8 Nov 2021 17:32:30 +0900
Subject: [PATCH v13 1/3] Support node initialization from backup with
 tablespaces

User defined tablespaces appear as symlinks in in the backup.  This
commit tweaks recursive copy subroutine to allow for symlinks specific
to tablespaces.
---
 src/test/perl/PostgreSQL/Test/Cluster.pm      | 29 +++++++++++-
 .../perl/PostgreSQL/Test/RecursiveCopy.pm     | 45 ++++++++++++++++---
 2 files changed, 66 insertions(+), 8 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 9467a199c8..19a667ebe4 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -634,6 +634,32 @@ sub backup_fs_cold
 	return;
 }
 
+sub _srcsymlink
+{
+	my ($srcpath, $destpath) = @_;
+
+	croak "Cannot operate on symlink \"$srcpath\""
+		if ($srcpath !~ qr{/(pg_tblspc/[0-9]+)$});
+
+	# We have mapped tablespaces. Copy them individually
+	my $tmpdir = PostgreSQL::Test::Utils::tempdir();
+	my $dstrealdir = PostgreSQL::Test::Utils::perl2host($tmpdir);
+	my $srcrealdir = readlink($srcpath);
+
+	opendir(my $dh, $srcrealdir);
+	while (my $entry = (readdir $dh))
+	{
+		next if ($entry eq '.' or $entry eq '..');
+		my $spath = "$srcrealdir/$entry";
+		my $dpath = "$dstrealdir/$entry";
+		PostgreSQL::Test::RecursiveCopy::copypath($spath, $dpath);
+	}
+	closedir $dh;
+
+	symlink $dstrealdir, $destpath;
+
+	return 1;
+}
 
 # Common sub of backup_fs_hot and backup_fs_cold
 sub _backup_fs
@@ -743,7 +769,8 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path,
+								srcsymlinkfn => \&_srcsymlink);
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/RecursiveCopy.pm b/src/test/perl/PostgreSQL/Test/RecursiveCopy.pm
index dd320a605e..2a636cef84 100644
--- a/src/test/perl/PostgreSQL/Test/RecursiveCopy.pm
+++ b/src/test/perl/PostgreSQL/Test/RecursiveCopy.pm
@@ -49,6 +49,11 @@ This subroutine will be called for each entry in the source directory with its
 relative path as only parameter; if the subroutine returns true the entry is
 copied, otherwise the file is skipped.
 
+If the B<srcsymlinkfn> parameter is given, it must be a subroutine reference.
+This subroutine will be called when the source directory is a symlink. It
+determines the mechanism that copies files from the source directory to the
+destination directory.
+
 On failure the target directory may be in some incomplete state; no cleanup is
 attempted.
 
@@ -68,6 +73,7 @@ sub copypath
 {
 	my ($base_src_dir, $base_dest_dir, %params) = @_;
 	my $filterfn;
+	my $srcsymlinkfn;
 
 	if (defined $params{filterfn})
 	{
@@ -82,31 +88,55 @@ sub copypath
 		$filterfn = sub { return 1; };
 	}
 
+	if (defined $params{srcsymlinkfn})
+	{
+		croak "if specified, srcsymlinkfn must be a subroutine reference"
+			unless defined(ref $params{srcsymlinkfn})
+			and (ref $params{srcsymlinkfn} eq 'CODE');
+
+		$srcsymlinkfn = $params{srcsymlinkfn};
+	}
+	else
+	{
+		$srcsymlinkfn = undef;
+	}
+
 	# Complain if original path is bogus, because _copypath_recurse won't.
 	croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir;
 
 	# Start recursive copy from current directory
-	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn);
+	return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn, $srcsymlinkfn);
 }
 
 # Recursive private guts of copypath
 sub _copypath_recurse
 {
-	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_;
+	my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn,
+		$srcsymlinkfn) = @_;
 	my $srcpath  = "$base_src_dir/$curr_path";
 	my $destpath = "$base_dest_dir/$curr_path";
 
 	# invoke the filter and skip all further operation if it returns false
 	return 1 unless &$filterfn($curr_path);
 
-	# Check for symlink -- needed only on source dir
-	# (note: this will fall through quietly if file is already gone)
-	croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath;
-
 	# Abort if destination path already exists.  Should we allow directories
 	# to exist already?
 	croak "Destination path \"$destpath\" already exists" if -e $destpath;
 
+	# Check for symlink -- needed only on source dir
+	# If caller provided us with a callback, call it; otherwise we're out.
+	if (-l $srcpath)
+	{
+		if (defined $srcsymlinkfn)
+		{
+			return &$srcsymlinkfn($srcpath, $destpath);
+		}
+		else
+		{
+			croak "Cannot operate on symlink \"$srcpath\"";
+		}
+	}
+
 	# If this source path is a file, simply copy it to destination with the
 	# same name and we're done.
 	if (-f $srcpath)
@@ -139,7 +169,8 @@ sub _copypath_recurse
 		{
 			next if ($entry eq '.' or $entry eq '..');
 			_copypath_recurse($base_src_dir, $base_dest_dir,
-				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn)
+				$curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn,
+				$srcsymlinkfn)
 			  or die "copypath $srcpath/$entry -> $destpath/$entry failed";
 		}
 
-- 
2.27.0

From 030f30d330dba3a6c3ff3f9561348375d30a1486 Mon Sep 17 00:00:00 2001
From: Asim R P <apraveen@pivotal.io>
Date: Fri, 20 Sep 2019 17:31:25 +0530
Subject: [PATCH v13 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 162 +++++++++++++++++++++-
 1 file changed, 161 insertions(+), 1 deletion(-)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index d7806e6671..a4e1fcb5dc 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -9,9 +9,10 @@ use warnings;
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
+use File::Path qw(rmtree);
 use Config;
 
-plan tests => 3;
+plan tests => 5;
 
 my $node = PostgreSQL::Test::Cluster->new('primary');
 $node->init(allows_streaming => 1);
@@ -62,3 +63,162 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
+
+# TEST 4
+#
+# Ensure that a missing tablespace directory during crash recovery on
+# a standby is handled correctly.  The standby should finish crash
+# recovery successfully because a matching drop database record is
+# found in the WAL.  The following scnearios are covered:
+#
+# 1. Create a database against a user-defined tablespace then drop the
+#    database.
+#
+# 2. Create a database against a user-defined tablespace then drop the
+#    database and the tablespace.
+#
+# 3. Move a database from source tablespace to target tablespace then
+#    drop the source tablespace.
+#
+# 4. Create a database from another database as template then drop the
+#    template database.
+#
+#
+
+my $node_master = PostgreSQL::Test::Cluster->new('master2');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $dropme_ts_master1 = PostgreSQL::Test::Utils::tempdir();
+$dropme_ts_master1 = PostgreSQL::Test::Utils::perl2host($dropme_ts_master1);
+my $dropme_ts_master2 = PostgreSQL::Test::Utils::tempdir();
+$dropme_ts_master2 = PostgreSQL::Test::Utils::perl2host($dropme_ts_master2);
+my $source_ts_master = PostgreSQL::Test::Utils::tempdir();
+$source_ts_master = PostgreSQL::Test::Utils::perl2host($source_ts_master);
+my $target_ts_master = PostgreSQL::Test::Utils::tempdir();
+$target_ts_master = PostgreSQL::Test::Utils::perl2host($target_ts_master);
+
+$node_master->safe_psql('postgres',
+						qq[CREATE TABLESPACE dropme_ts1 location '$dropme_ts_master1';
+						   CREATE TABLESPACE dropme_ts2 location '$dropme_ts_master2';
+						   CREATE TABLESPACE source_ts location '$source_ts_master';
+						   CREATE TABLESPACE target_ts location '$target_ts_master';
+						   CREATE DATABASE template_db IS_TEMPLATE = true;]);
+
+my $dropme_ts_standby1 = PostgreSQL::Test::Utils::tempdir();
+$dropme_ts_standby1 = PostgreSQL::Test::Utils::perl2host($dropme_ts_standby1);
+my $dropme_ts_standby2 = PostgreSQL::Test::Utils::tempdir();
+$dropme_ts_standby2 = PostgreSQL::Test::Utils::perl2host($dropme_ts_standby2);
+my $source_ts_standby = PostgreSQL::Test::Utils::tempdir();
+$source_ts_standby = PostgreSQL::Test::Utils::perl2host($source_ts_standby);
+my $target_ts_standby = PostgreSQL::Test::Utils::tempdir();
+$target_ts_standby = PostgreSQL::Test::Utils::perl2host($target_ts_standby);
+
+# Take backup
+my $backup_name = 'my_backup';
+my $ts_mapping = [ "--tablespace-mapping=$dropme_ts_master1=$dropme_ts_standby1",
+  "--tablespace-mapping=$dropme_ts_master2=$dropme_ts_standby2",
+  "--tablespace-mapping=$source_ts_master=$source_ts_standby",
+  "--tablespace-mapping=$target_ts_master=$target_ts_standby" ];
+$node_master->backup($backup_name, backup_options => $ts_mapping);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_master->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+# Make sure to perform restartpoint after tablespace creation
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_master->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+$node_master->wait_for_catchup($node_standby, 'replay',
+							   $node_master->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_master = PostgreSQL::Test::Cluster->new('master3');
+$node_master->init(allows_streaming => 1);
+$node_master->start;
+
+# Create tablespace
+my $ts_master = PostgreSQL::Test::Utils::tempdir();
+$ts_master = PostgreSQL::Test::Utils::perl2host($ts_master);
+$node_master->safe_psql('postgres', "CREATE TABLESPACE ts1 LOCATION '$ts_master'");
+$node_master->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+my $ts_standby = PostgreSQL::Test::Utils::tempdir("standby");
+$ts_standby = PostgreSQL::Test::Utils::perl2host($ts_standby);
+
+# Take backup
+$backup_name = 'my_backup';
+$node_master->backup($backup_name,
+					 backup_options =>
+					   [ "--tablespace-mapping=$ts_master=$ts_standby" ]);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_master, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+#
+# The tablespace mapping is lost when the standby node is initialized
+# from basebackup because RecursiveCopy::copypath creates a new temp
+# directory for each tablspace symlink found in backup.  We must
+# obtain the correct tablespace directory by querying standby.
+$ts_standby = $node_standby->safe_psql(
+	'postgres',
+	"select pg_tablespace_location(oid) from pg_tablespace where spcname = 'ts1'");
+rmtree($ts_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_master->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_master->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.27.0

From 42d379e23f99b91565c24e23073a6da14bf98f19 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v13 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 +
 src/backend/access/transam/xlogutils.c | 145 +++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  55 ++++++++++
 src/backend/commands/tablespace.c      |   5 +
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 215 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5cda30836f..c6d5fc782f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8313,6 +8313,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b33e0531ed..99abf8b2f4 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 029fab48df..0f483edb71 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2143,7 +2143,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2161,6 +2163,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2218,6 +2269,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 4b96eec9df..0d5dfe007f 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -1527,6 +1527,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index eebc91f3a5..3341efc052 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From 5381df72dff0f326ffd20ae212bc43aa54ee8a86 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 11 Nov 2021 20:42:00 +0900
Subject: [PATCH v14 1/3] Add tablespace support to TAP framework

TAP framework doesn't support nodes that have tablespaces.  Especially
backup and initialization from backups failed if the source node has
tablespaces.  This commit provides simple way to create tablespace
directories and allows backup routines to handle tablespaces.
---
 src/test/perl/PostgreSQL/Test/Cluster.pm | 262 ++++++++++++++++++++++-
 src/test/perl/PostgreSQL/Test/Utils.pm   |  43 ++++
 2 files changed, 303 insertions(+), 2 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 9467a199c8..e195f11a23 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -287,6 +287,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = PostgreSQL::Test::Utils::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -302,6 +360,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a hash from tablespace OID to tablespace directory name of
+tablespace directories that the specified backup has.  For example, an
+oid 16384 pointing to ../tsps/backup1/ts1 is stored as $hash{16384} =
+"ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->install_path()
 
 The configured install path (if any) for the node.
@@ -334,6 +463,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	print $fh "Install Path: ", $self->{_install_path} . "\n"
@@ -564,6 +694,43 @@ sub adjust_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -583,9 +750,24 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
 
 	local %ENV = $self->_get_env();
 
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = PostgreSQL::Test::Utils::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
+
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	PostgreSQL::Test::Utils::system_or_bail(
 		'pg_basebackup', '-D',
@@ -593,7 +775,32 @@ sub backup
 		$self->host,     '-p',
 		$self->port,     '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup directory.
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -655,11 +862,32 @@ sub _backup_fs
 	PostgreSQL::Test::RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -743,7 +971,37 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_path,
+			$data_path,
+			# Skipping tablespace symlinks
+			filterfn => sub {
+				my $src = shift;
+				return ($src !~ m!^pg_tblspc/[0-9]+$!);
+			});
+	}
+
+	# Copy tablespaces if any
+	my %tsps = $root_node->backup_tablespaces($backup_name);
+	if (%tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %tsps)
+		{
+			my $tsp = $tsps{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			PostgreSQL::Test::RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			PostgreSQL::Test::Utils::dir_symlink($tspdst, $linkdst);
+		}
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index f29d43f1f3..c3b5b4af34 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -725,6 +725,49 @@ sub dir_symlink
 
 =pod
 
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name = perl2host($name);
+		$name .= '/..';
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
+=pod
+
 =back
 
 =head1 Test::More-LIKE METHODS
-- 
2.27.0

From 69ca7d3657d9ea20d00a6486b0102899a4739a08 Mon Sep 17 00:00:00 2001
From: P <apraveen@pivotal.io>
Date: Thu, 11 Nov 2021 20:46:17 +0900
Subject: [PATCH v14 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 107 +++++++++++++++++++++-
 1 file changed, 106 insertions(+), 1 deletion(-)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index d7806e6671..44254a7257 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -11,7 +11,7 @@ use PostgreSQL::Test::Utils;
 use Test::More;
 use Config;
 
-plan tests => 3;
+plan tests => 5;
 
 my $node = PostgreSQL::Test::Cluster->new('primary');
 $node->init(allows_streaming => 1);
@@ -62,3 +62,108 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = PostgreSQL::Test::Cluster->new('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.27.0

From 948e08998f106da269e792ca67b3aa8a22a258eb Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v14 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 +
 src/backend/access/transam/xlogutils.c | 145 +++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  55 ++++++++++
 src/backend/commands/tablespace.c      |   5 +
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 215 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e073121a7e..badda1deb2 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8309,6 +8309,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b33e0531ed..99abf8b2f4 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 029fab48df..0f483edb71 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2143,7 +2143,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2161,6 +2163,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2218,6 +2269,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 4b96eec9df..0d5dfe007f 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -1527,6 +1527,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index eebc91f3a5..3341efc052 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From 5f88a80b9a585ca611ab6424f035330a47b2449f Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 11 Nov 2021 20:42:00 +0900
Subject: [PATCH v15 1/3] Add tablespace support to TAP framework

TAP framework doesn't support nodes that have tablespaces.  Especially
backup and initialization from backups failed if the source node has
tablespaces.  This commit provides simple way to create tablespace
directories and allows backup routines to handle tablespaces.
---
 src/bin/pg_basebackup/t/010_pg_basebackup.pl |   2 +-
 src/test/perl/PostgreSQL/Test/Cluster.pm     | 264 ++++++++++++++++++-
 src/test/perl/PostgreSQL/Test/Utils.pm       |  43 +++
 3 files changed, 306 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index f0243f28d4..c139b5e000 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -257,7 +257,7 @@ $node->safe_psql('postgres',
 	"CREATE TABLESPACE tblspc1 LOCATION '$realTsDir';");
 $node->safe_psql('postgres',
 	    "CREATE TABLE test1 (a int) TABLESPACE tblspc1;"
-	  . "INSERT INTO test1 VALUES (1234);");
+				 . "INSERT INTO test1 VALUES (1234);");
 $node->backup('tarbackup2', backup_options => ['-Ft']);
 # empty test1, just so that it's different from the to-be-restored data
 $node->safe_psql('postgres', "TRUNCATE TABLE test1;");
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index b7d4c24553..d433ccf610 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -298,6 +298,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = PostgreSQL::Test::Utils::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -313,6 +371,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a reference to hash from tablespace OID to tablespace
+directory name of tablespace directory that the specified backup has.
+For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return \%ret;
+}
+
+=pod
+
 =item $node->install_path()
 
 The configured install path (if any) for the node.
@@ -345,6 +474,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	print $fh "Install Path: ", $self->{_install_path} . "\n"
@@ -575,6 +705,43 @@ sub adjust_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -594,9 +761,24 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
 
 	local %ENV = $self->_get_env();
 
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = PostgreSQL::Test::Utils::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
+
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	PostgreSQL::Test::Utils::system_or_bail(
 		'pg_basebackup', '-D',
@@ -604,7 +786,33 @@ sub backup
 		$self->host,     '-p',
 		$self->port,     '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup
+	# directory, unless the backup is in tar mode.
+	if (%tsps && ! -f "$backup_path/base.tar")
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -666,11 +874,32 @@ sub _backup_fs
 	PostgreSQL::Test::RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -754,7 +983,38 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_path,
+			$data_path,
+			# Skipping tablespace symlinks
+			filterfn => sub {
+				my $src = shift;
+				return ($src !~ m!^pg_tblspc/[0-9]+$!);
+			});
+	}
+
+	# Copy tablespaces if any
+	my $tsps = $root_node->backup_tablespaces($backup_name);
+
+	if ($tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %{$tsps})
+		{
+			my $tsp = ${$tsps}{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			PostgreSQL::Test::RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			PostgreSQL::Test::Utils::dir_symlink($tspdst, $linkdst);
+		}
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 50be10fb5a..266f1c5aaf 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -724,6 +724,49 @@ sub dir_symlink
 
 =pod
 
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name = perl2host($name);
+		$name .= '/..';
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
+=pod
+
 =back
 
 =head1 Test::More-LIKE METHODS
-- 
2.27.0

From 0d4b1968c7bed11d47b169e8d4e2929db75c38b8 Mon Sep 17 00:00:00 2001
From: P <apraveen@pivotal.io>
Date: Thu, 11 Nov 2021 20:46:17 +0900
Subject: [PATCH v15 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 107 +++++++++++++++++++++-
 1 file changed, 106 insertions(+), 1 deletion(-)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 3892aba3e5..421cf52dfe 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -11,7 +11,7 @@ use PostgreSQL::Test::Utils;
 use Test::More;
 use Config;
 
-plan tests => 3;
+plan tests => 5;
 
 my $node = PostgreSQL::Test::Cluster->new('primary');
 $node->init(allows_streaming => 1);
@@ -62,3 +62,108 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = PostgreSQL::Test::Cluster->new('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.27.0

From d82536792a6544fa082d4cde021e87f44854a2eb Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v15 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 +
 src/backend/access/transam/xlogutils.c | 145 +++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  55 ++++++++++
 src/backend/commands/tablespace.c      |   5 +
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 215 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c9d4cbf3ff..ec279c6158 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8314,6 +8314,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 90e1c48390..cd00e0f01e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 509d1a3e92..02b080e4ef 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2143,7 +2143,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2161,6 +2163,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2218,6 +2269,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index b2ccf5e06e..b2975a0bd2 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -1565,6 +1565,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..5d9c20cae7 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From 0423d2b9aae0620c07b522632a8074ecd8ffef64 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 11 Nov 2021 20:42:00 +0900
Subject: [PATCH v15 1/3] Add tablespace support to TAP framework

TAP framework doesn't support nodes that have tablespaces.  Especially
backup and initialization from backups failed if the source node has
tablespaces.  This commit provides simple way to create tablespace
directories and allows backup routines to handle tablespaces.
---
 src/bin/pg_basebackup/t/010_pg_basebackup.pl |   2 +-
 src/test/perl/PostgreSQL/Test/Cluster.pm     | 264 ++++++++++++++++++-
 src/test/perl/PostgreSQL/Test/Utils.pm       |  42 +++
 3 files changed, 305 insertions(+), 3 deletions(-)

diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index f0243f28d4..c139b5e000 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -257,7 +257,7 @@ $node->safe_psql('postgres',
 	"CREATE TABLESPACE tblspc1 LOCATION '$realTsDir';");
 $node->safe_psql('postgres',
 	    "CREATE TABLE test1 (a int) TABLESPACE tblspc1;"
-	  . "INSERT INTO test1 VALUES (1234);");
+				 . "INSERT INTO test1 VALUES (1234);");
 $node->backup('tarbackup2', backup_options => ['-Ft']);
 # empty test1, just so that it's different from the to-be-restored data
 $node->safe_psql('postgres', "TRUNCATE TABLE test1;");
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index b7d4c24553..d433ccf610 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -298,6 +298,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = PostgreSQL::Test::Utils::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -313,6 +371,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a reference to hash from tablespace OID to tablespace
+directory name of tablespace directory that the specified backup has.
+For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return \%ret;
+}
+
+=pod
+
 =item $node->install_path()
 
 The configured install path (if any) for the node.
@@ -345,6 +474,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	print $fh "Install Path: ", $self->{_install_path} . "\n"
@@ -575,6 +705,43 @@ sub adjust_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -594,9 +761,24 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
 
 	local %ENV = $self->_get_env();
 
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = PostgreSQL::Test::Utils::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
+
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	PostgreSQL::Test::Utils::system_or_bail(
 		'pg_basebackup', '-D',
@@ -604,7 +786,33 @@ sub backup
 		$self->host,     '-p',
 		$self->port,     '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup
+	# directory, unless the backup is in tar mode.
+	if (%tsps && ! -f "$backup_path/base.tar")
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -666,11 +874,32 @@ sub _backup_fs
 	PostgreSQL::Test::RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -754,7 +983,38 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_path,
+			$data_path,
+			# Skipping tablespace symlinks
+			filterfn => sub {
+				my $src = shift;
+				return ($src !~ m!^pg_tblspc/[0-9]+$!);
+			});
+	}
+
+	# Copy tablespaces if any
+	my $tsps = $root_node->backup_tablespaces($backup_name);
+
+	if ($tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %{$tsps})
+		{
+			my $tsp = ${$tsps}{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			PostgreSQL::Test::RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			PostgreSQL::Test::Utils::dir_symlink($tspdst, $linkdst);
+		}
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 50be10fb5a..e0e5956e9b 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -724,6 +724,48 @@ sub dir_symlink
 
 =pod
 
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name = perl2host($name);
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
+=pod
+
 =back
 
 =head1 Test::More-LIKE METHODS
-- 
2.27.0

From e2772ce12fac4b552f26ad5c1c694766d3429170 Mon Sep 17 00:00:00 2001
From: "apraveen@pivotal.io" <apraveen@pivotal.io>
Date: Thu, 11 Nov 2021 20:46:17 +0900
Subject: [PATCH v15 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 107 +++++++++++++++++++++-
 1 file changed, 106 insertions(+), 1 deletion(-)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 3892aba3e5..421cf52dfe 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -11,7 +11,7 @@ use PostgreSQL::Test::Utils;
 use Test::More;
 use Config;
 
-plan tests => 3;
+plan tests => 5;
 
 my $node = PostgreSQL::Test::Cluster->new('primary');
 $node->init(allows_streaming => 1);
@@ -62,3 +62,108 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = PostgreSQL::Test::Cluster->new('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
-- 
2.27.0

From ad9266ffdf4fdfc850218d6bb558127a2753f05c Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v15 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlog.c      |   6 +
 src/backend/access/transam/xlogutils.c | 145 +++++++++++++++++++++++++
 src/backend/commands/dbcommands.c      |  55 ++++++++++
 src/backend/commands/tablespace.c      |   5 +
 src/include/access/xlogutils.h         |   4 +
 5 files changed, 215 insertions(+)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c9d4cbf3ff..ec279c6158 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8314,6 +8314,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 90e1c48390..cd00e0f01e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 509d1a3e92..02b080e4ef 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2143,7 +2143,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2161,6 +2163,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2218,6 +2269,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index b2ccf5e06e..b2975a0bd2 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -1565,6 +1565,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..5d9c20cae7 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From 35958b17c62cd14f81efa26a097c32c273028f77 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 11 Nov 2021 20:42:00 +0900
Subject: [PATCH v16 1/3] Add tablespace support to TAP framework

TAP framework doesn't support nodes that have tablespaces.  Especially
backup and initialization from backups failed if the source node has
tablespaces.  This commit provides simple way to create tablespace
directories and allows backup routines to handle tablespaces.
---
 src/test/perl/PostgreSQL/Test/Cluster.pm | 264 ++++++++++++++++++++++-
 src/test/perl/PostgreSQL/Test/Utils.pm   |  43 ++++
 2 files changed, 305 insertions(+), 2 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index be05845248..15d57b9a71 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -298,6 +298,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = PostgreSQL::Test::Utils::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -313,6 +371,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a reference to hash from tablespace OID to tablespace
+directory name of tablespace directory that the specified backup has.
+For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return \%ret;
+}
+
+=pod
+
 =item $node->install_path()
 
 The configured install path (if any) for the node.
@@ -370,6 +499,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	print $fh "Install Path: ", $self->{_install_path} . "\n"
@@ -600,6 +730,43 @@ sub adjust_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -619,9 +786,24 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
 
 	local %ENV = $self->_get_env();
 
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = PostgreSQL::Test::Utils::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
+
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	PostgreSQL::Test::Utils::system_or_bail(
 		'pg_basebackup', '-D',
@@ -629,7 +811,33 @@ sub backup
 		$self->host,     '-p',
 		$self->port,     '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup
+	# directory, unless the backup is in tar mode.
+	if (%tsps && ! -f "$backup_path/base.tar")
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -691,11 +899,32 @@ sub _backup_fs
 	PostgreSQL::Test::RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -779,7 +1008,38 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_path,
+			$data_path,
+			# Skipping tablespace symlinks
+			filterfn => sub {
+				my $src = shift;
+				return ($src !~ m!^pg_tblspc/[0-9]+$!);
+			});
+	}
+
+	# Copy tablespaces if any
+	my $tsps = $root_node->backup_tablespaces($backup_name);
+
+	if ($tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %{$tsps})
+		{
+			my $tsp = ${$tsps}{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			PostgreSQL::Test::RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			PostgreSQL::Test::Utils::dir_symlink($tspdst, $linkdst);
+		}
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 46cd746796..6daac4ebdf 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -711,6 +711,49 @@ sub dir_symlink
 
 =pod
 
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name = perl2host($name);
+		$name .= '/..';
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
+=pod
+
 =back
 
 =head1 Test::More-LIKE METHODS
-- 
2.27.0

From 1f3d614bf6462876fd90166fcd9b7bf7e30e8c9e Mon Sep 17 00:00:00 2001
From: P <apraveen@pivotal.io>
Date: Thu, 11 Nov 2021 20:46:17 +0900
Subject: [PATCH v16 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 105 ++++++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 14154d1ce0..1998a321da 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -61,4 +61,109 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
 
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = PostgreSQL::Test::Cluster->new('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
+
 done_testing();
-- 
2.27.0

From b353dc4259baf022de2f6ce9a0301bf812d02ef2 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v16 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlogrecovery.c |   6 +
 src/backend/access/transam/xlogutils.c    | 145 ++++++++++++++++++++++
 src/backend/commands/dbcommands.c         |  56 +++++++++
 src/backend/commands/tablespace.c         |   6 +
 src/include/access/xlogutils.h            |   4 +
 5 files changed, 217 insertions(+)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f9f212680b..97fed1e04d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2043,6 +2043,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 54d5f20734..3f8f7dadac 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index c37e3c9a9a..8994e9da99 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -2382,7 +2383,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2400,6 +2403,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2462,6 +2514,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..62ee0ca978 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -57,6 +57,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -1574,6 +1575,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..5d9c20cae7 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From bb714659adcde5265974c46b061966e5dfc556be Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 11 Nov 2021 20:42:00 +0900
Subject: [PATCH v17 1/3] Add tablespace support to TAP framework

TAP framework doesn't support nodes that have tablespaces.  Especially
backup and initialization from backups failed if the source node has
tablespaces.  This commit provides simple way to create tablespace
directories and allows backup routines to handle tablespaces.
---
 src/test/perl/PostgreSQL/Test/Cluster.pm | 264 ++++++++++++++++++++++-
 src/test/perl/PostgreSQL/Test/Utils.pm   |  42 ++++
 2 files changed, 304 insertions(+), 2 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index be05845248..15d57b9a71 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -298,6 +298,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = PostgreSQL::Test::Utils::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -313,6 +371,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a reference to hash from tablespace OID to tablespace
+directory name of tablespace directory that the specified backup has.
+For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return \%ret;
+}
+
+=pod
+
 =item $node->install_path()
 
 The configured install path (if any) for the node.
@@ -370,6 +499,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	print $fh "Install Path: ", $self->{_install_path} . "\n"
@@ -600,6 +730,43 @@ sub adjust_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -619,9 +786,24 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
 
 	local %ENV = $self->_get_env();
 
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = PostgreSQL::Test::Utils::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
+
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	PostgreSQL::Test::Utils::system_or_bail(
 		'pg_basebackup', '-D',
@@ -629,7 +811,33 @@ sub backup
 		$self->host,     '-p',
 		$self->port,     '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup
+	# directory, unless the backup is in tar mode.
+	if (%tsps && ! -f "$backup_path/base.tar")
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -691,11 +899,32 @@ sub _backup_fs
 	PostgreSQL::Test::RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -779,7 +1008,38 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_path,
+			$data_path,
+			# Skipping tablespace symlinks
+			filterfn => sub {
+				my $src = shift;
+				return ($src !~ m!^pg_tblspc/[0-9]+$!);
+			});
+	}
+
+	# Copy tablespaces if any
+	my $tsps = $root_node->backup_tablespaces($backup_name);
+
+	if ($tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %{$tsps})
+		{
+			my $tsp = ${$tsps}{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			PostgreSQL::Test::RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			PostgreSQL::Test::Utils::dir_symlink($tspdst, $linkdst);
+		}
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 46cd746796..7f440c4662 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -711,6 +711,48 @@ sub dir_symlink
 
 =pod
 
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name .= '/..';
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
+=pod
+
 =back
 
 =head1 Test::More-LIKE METHODS
-- 
2.27.0

From 21da10267102dafe44afa2d99d8969a747e8e072 Mon Sep 17 00:00:00 2001
From: P <apraveen@pivotal.io>
Date: Thu, 11 Nov 2021 20:46:17 +0900
Subject: [PATCH v17 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 105 ++++++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 14154d1ce0..1998a321da 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -61,4 +61,109 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
 
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = PostgreSQL::Test::Cluster->new('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
+
 done_testing();
-- 
2.27.0

From 79201550374497555bf272a84c810e116412df3b Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v17 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlogrecovery.c |   6 +
 src/backend/access/transam/xlogutils.c    | 145 ++++++++++++++++++++++
 src/backend/commands/dbcommands.c         |  56 +++++++++
 src/backend/commands/tablespace.c         |   6 +
 src/include/access/xlogutils.h            |   4 +
 5 files changed, 217 insertions(+)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f9f212680b..97fed1e04d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2043,6 +2043,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 54d5f20734..3f8f7dadac 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index c37e3c9a9a..8994e9da99 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -2382,7 +2383,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2400,6 +2403,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2462,6 +2514,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..62ee0ca978 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -57,6 +57,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -1574,6 +1575,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..5d9c20cae7 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From ee17f0f4400ce484cdba80c84744ae47d68c6fa4 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 11 Nov 2021 20:42:00 +0900
Subject: [PATCH v18 1/3] Add tablespace support to TAP framework

TAP framework doesn't support nodes that have tablespaces.  Especially
backup and initialization from backups failed if the source node has
tablespaces.  This commit provides simple way to create tablespace
directories and allows backup routines to handle tablespaces.
---
 src/test/perl/PostgreSQL/Test/Cluster.pm | 264 ++++++++++++++++++++++-
 src/test/perl/PostgreSQL/Test/Utils.pm   |  42 ++++
 2 files changed, 304 insertions(+), 2 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index be05845248..15d57b9a71 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -298,6 +298,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = PostgreSQL::Test::Utils::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -313,6 +371,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a reference to hash from tablespace OID to tablespace
+directory name of tablespace directory that the specified backup has.
+For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = PostgreSQL::Test::Utils::dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return \%ret;
+}
+
+=pod
+
 =item $node->install_path()
 
 The configured install path (if any) for the node.
@@ -370,6 +499,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	print $fh "Install Path: ", $self->{_install_path} . "\n"
@@ -600,6 +730,43 @@ sub adjust_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -619,9 +786,24 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
 
 	local %ENV = $self->_get_env();
 
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = PostgreSQL::Test::Utils::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
+
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	PostgreSQL::Test::Utils::system_or_bail(
 		'pg_basebackup', '-D',
@@ -629,7 +811,33 @@ sub backup
 		$self->host,     '-p',
 		$self->port,     '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup
+	# directory, unless the backup is in tar mode.
+	if (%tsps && ! -f "$backup_path/base.tar")
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -691,11 +899,32 @@ sub _backup_fs
 	PostgreSQL::Test::RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			PostgreSQL::Test::Utils::dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -779,7 +1008,38 @@ sub init_from_backup
 	else
 	{
 		rmdir($data_path);
-		PostgreSQL::Test::RecursiveCopy::copypath($backup_path, $data_path);
+		PostgreSQL::Test::RecursiveCopy::copypath(
+			$backup_path,
+			$data_path,
+			# Skipping tablespace symlinks
+			filterfn => sub {
+				my $src = shift;
+				return ($src !~ m!^pg_tblspc/[0-9]+$!);
+			});
+	}
+
+	# Copy tablespaces if any
+	my $tsps = $root_node->backup_tablespaces($backup_name);
+
+	if ($tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %{$tsps})
+		{
+			my $tsp = ${$tsps}{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			PostgreSQL::Test::RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			PostgreSQL::Test::Utils::dir_symlink($tspdst, $linkdst);
+		}
 	}
 	chmod(0700, $data_path);
 
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index 46cd746796..7f440c4662 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -711,6 +711,48 @@ sub dir_symlink
 
 =pod
 
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name .= '/..';
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
+=pod
+
 =back
 
 =head1 Test::More-LIKE METHODS
-- 
2.27.0

From 907d295d4e9823b8c51818272b10f0fe518eff8e Mon Sep 17 00:00:00 2001
From: P <apraveen@pivotal.io>
Date: Thu, 11 Nov 2021 20:46:17 +0900
Subject: [PATCH v18 2/3] Tests to replay create database operation on standby

The tests demonstrate that standby fails to replay a create database
WAL record during crash recovery, if one or more of underlying
directories are missing from the file system.  This can happen if a
drop tablespace or drop database WAL record has been replayed in
archive recovery, before a crash.  And then the create database record
happens to be replayed again during crash recovery.  The failures
indicate bugs that need to be fixed.

The first test, TEST 4, performs several DDL operations resulting in a
database directory being removed, along with a few create database
operations.  It expects crash recovery to succeed because for each
missing directory encountered during create database replay, a matching
drop tablespace or drop database WAL record is found later.

Second test, TEST 5, validates that a standby rightfully aborts replay
during archive recovery, if a missing directory is encountered when
replaying create database WAL record.

These tests have been proposed and implemented in various ways by
Alexandra Wang, Anastasia Lubennikova, Kyotaro Horiguchi, Paul Guo and me.
---
 src/test/recovery/t/011_crash_recovery.pl | 105 ++++++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 14154d1ce0..1998a321da 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -61,4 +61,109 @@ is($node->safe_psql('postgres', qq[SELECT pg_xact_status('$xid');]),
 $stdin .= "\\q\n";
 $tx->finish;    # wait for psql to quit gracefully
 
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = PostgreSQL::Test::Cluster->new('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
+
 done_testing();
-- 
2.27.0

From 5d335b2b62d9c38b5fd80895c839efc843a5de6d Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 9 Jan 2020 17:54:40 -0300
Subject: [PATCH v18 3/3] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlogrecovery.c |   6 +
 src/backend/access/transam/xlogutils.c    | 145 ++++++++++++++++++++++
 src/backend/commands/dbcommands.c         |  56 +++++++++
 src/backend/commands/tablespace.c         |   6 +
 src/include/access/xlogutils.h            |   4 +
 5 files changed, 217 insertions(+)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f9f212680b..97fed1e04d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2043,6 +2043,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 54d5f20734..3f8f7dadac 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index c37e3c9a9a..8994e9da99 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -2382,7 +2383,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2400,6 +2403,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2462,6 +2514,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..62ee0ca978 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -57,6 +57,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -1574,6 +1575,11 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..5d9c20cae7 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
-- 
2.27.0

From 43bb3ba8900edd53a1feb0acb1a72bdc22bb1627 Mon Sep 17 00:00:00 2001
From: P <apraveen@pivotal.io>
Date: Mon, 7 Mar 2022 17:10:07 +0900
Subject: [PATCH v20] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Bug identified by Paul Guo.

Authored by Paul Guo, Kyotaro Horiguchi and Asim R P.
---
 src/backend/access/transam/xlogrecovery.c   |   6 +
 src/backend/access/transam/xlogutils.c      | 145 ++++++++++++++++++++
 src/backend/commands/dbcommands.c           |  56 ++++++++
 src/backend/commands/tablespace.c           |  16 +++
 src/include/access/xlogutils.h              |   4 +
 src/test/recovery/t/029_replay_tsp_drops.pl |  62 +++++++++
 6 files changed, 289 insertions(+)
 create mode 100644 src/test/recovery/t/029_replay_tsp_drops.pl

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f9f212680b..97fed1e04d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2043,6 +2043,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 54d5f20734..3f8f7dadac 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -79,6 +79,151 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid spcNode;
+	Oid dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+void
+XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									   100,
+									   &ctl,
+									   HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "missing directory %s (tablespace %d) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG2, "missing directory %s (tablespace %d database %d) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG2, "logged missing dir %s (tablespace %d)",
+				 path, spcNode);
+		else
+			elog(DEBUG2, "logged missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+	}
+}
+
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %d)", spcNode);
+		}
+		else
+		{
+			char *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %d database %d)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %d database %d",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
 
 /* Report a reference to an invalid page */
 static void
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index c37e3c9a9a..8994e9da99 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -2382,7 +2383,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool	    skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2400,6 +2403,55 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that drop tablespace record appearing later in
+			 * the WAL as already been replayed.  That means we are replaying
+			 * the create database record second time, as part of crash
+			 * recovery.  In that case, the tablespace directory has already
+			 * been removed and the create database operation cannot be
+			 * replayed.  We should skip the replay but remember the missing
+			 * tablespace directory, to be matched with a drop tablespace
+			 * record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogReportMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping create database WAL record"),
+						 errdetail("Target tablespace \"%s\" not found. We "
+								   "expect to encounter a WAL record that "
+								   "removes this directory before reaching "
+								   "consistent state.", parent_path)));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * Source directory may be missing.  E.g. the template database used
+		 * for creating this database may have been dropped, due to reasons
+		 * noted above.  Moving a database from one tablespace may also be a
+		 * partner in the crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogReportMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping create database WAL record"),
+					 errdetail("Source database \"%s\" not found. We expect "
+							   "to encounter a WAL record that removes this "
+							   "directory before reaching consistent state.",
+							   src_path)));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2462,6 +2514,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..66bd28fc74 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -57,6 +57,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -1574,6 +1575,21 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		/*
+		 * Before we remove the tablespace directory, update minimum recovery
+		 * point to cover this WAL record. Once the tablespace is removed,
+		 * there's no going back.  This manually enforces the WAL-first rule.
+		 * Doing this before the removal means that if the removal fails for
+		 * some reason, the directory is left alone and needs to be manually
+		 * removed.  Alternatively you could update the minimum recovery point
+		 * after removal, but that would leave a small window where the
+		 * WAL-first rule could be violated.
+		 */
+		XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..5d9c20cae7 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogReportMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
diff --git a/src/test/recovery/t/029_replay_tsp_drops.pl b/src/test/recovery/t/029_replay_tsp_drops.pl
new file mode 100644
index 0000000000..de2a92661c
--- /dev/null
+++ b/src/test/recovery/t/029_replay_tsp_drops.pl
@@ -0,0 +1,62 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test recovery involving tablespace droppings.  If recovery stops
+# after once tablespace is removed, the next recovery should properly
+# ignore the operations within the removed tablespaces.
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+#use File::Compare;
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary1');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+$node_primary->psql('postgres',
+qq[
+	SET allow_in_place_tablespaces=on;
+	CREATE TABLESPACE dropme_ts1 LOCATION '';
+	CREATE TABLESPACE dropme_ts2 LOCATION '';
+	CREATE TABLESPACE source_ts  LOCATION '';
+	CREATE TABLESPACE target_ts  LOCATION '';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby1');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+# Ensure that a missing tablespace directory during create database
+done_testing();
-- 
2.27.0

diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index c37e3c9a9a..28aed8d296 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -47,6 +47,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -2382,6 +2383,7 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
@@ -2401,6 +2403,41 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		/*
+		 * It is possible that the tablespace was later dropped, but we are
+		 * re-redoing database create before that. In that case, those
+		 * directories are gone, and we do not create symlink.
+		 */
+		if (stat(dst_path, &st) < 0 && errno == ENOENT)
+		{
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			elog(WARNING, "creating missing directory: %s", parent_path);
+			if (stat(parent_path, &st) != 0 && pg_mkdir_p(parent_path, pg_dir_create_mode) != 0)
+			{
+				ereport(WARNING,
+						(errmsg("can not recursively create directory \"%s\"",
+								parent_path)));
+			}
+		}
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+		{
+			elog(WARNING, "creating missing copy source directory: %s", src_path);
+			if (stat(src_path, &st) != 0 && pg_mkdir_p(src_path, pg_dir_create_mode) != 0)
+			{
+				ereport(WARNING,
+						(errmsg("can not recursively create directory \"%s\"",
+								src_path)));
+			}
+		}
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..675f578dfe 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -155,8 +155,6 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -169,32 +167,8 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",

From 6a6fc73a93768a44ec026720c115f77c67d5cda2 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Mon, 21 Mar 2022 12:34:34 +0100
Subject: [PATCH v21] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds mechanism similar to invalid page hash table, to track
missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at
the end of crash recovery, the standby can safely enter archive
recovery.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 src/backend/access/transam/xlogrecovery.c   |   6 +
 src/backend/access/transam/xlogutils.c      | 159 +++++++++++++++++++-
 src/backend/commands/dbcommands.c           |  57 +++++++
 src/backend/commands/tablespace.c           |  17 +++
 src/include/access/xlogutils.h              |   4 +
 src/test/recovery/t/029_replay_tsp_drops.pl |  67 +++++++++
 src/tools/pgindent/typedefs.list            |   2 +
 7 files changed, 311 insertions(+), 1 deletion(-)
 create mode 100644 src/test/recovery/t/029_replay_tsp_drops.pl

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 9feea3e6ec..f48d8d51fb 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2043,6 +2043,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 511f2f186f..8c1b8216be 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -54,6 +54,164 @@ bool		InRecovery = false;
 /* Are we in Hot Standby mode? Only valid in startup process, see xlogutils.h */
 HotStandbyState standbyState = STANDBY_DISABLED;
 
+
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid			spcNode;
+	Oid			dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char		path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+
+/*
+ * Keep track of a directory that wasn't found while replaying database
+ * creation records.  These should match up with tablespace removal records
+ * later in the WAL stream; we verify that before reaching consistency.
+ */
+void
+XLogRememberMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool		found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									  100,
+									  &ctl,
+									  HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG1, "missing directory %s (tablespace %u) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG1, "missing directory %s (tablespace %u database %u) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG1, "logged missing dir %s (tablespace %u)",
+				 path, spcNode);
+		else
+			elog(DEBUG1, "logged missing dir %s (tablespace %u database %u)",
+				 path, spcNode, dbNode);
+	}
+}
+
+/*
+ * Remove an entry from the list of directories not found.  This is to be done
+ * when the matching tablespace removal WAL record is found.
+ */
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %u)", spcNode);
+		}
+		else
+		{
+			char	   *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %u database %u)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %u database %u",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
+
+
 /*
  * During XLOG replay, we may see XLOG records for incremental updates of
  * pages that no longer exist, because their relation was later dropped or
@@ -79,7 +237,6 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
-
 /* Report a reference to an invalid page */
 static void
 report_invalid_page(int elevel, RelFileNode node, ForkNumber forkno,
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 623e5ec778..95771b06a2 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -2483,7 +2484,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool		skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2501,6 +2504,56 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that a drop tablespace record appearing later in
+			 * WAL has already been replayed -- in other words, that we are
+			 * replaying the database creation record a second time with no
+			 * intervening checkpoint.  In that case, the tablespace directory
+			 * has already been removed and the create database operation
+			 * cannot be replayed.  Skip the replay itself, but remember the
+			 * fact that the tablespace directory is missing, to be matched
+			 * with the expected tablespace drop record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogRememberMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping replay of database creation WAL record"),
+						 errdetail("The target tablespace \"%s\" directory was not found.",
+								   parent_path),
+						 errhint("A future WAL record that removes the directory before reaching consistent mode is expected.")));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * If the source directory is missing, skip the copy and make a note of
+		 * it for later.
+		 *
+		 * One possible reason for this is that the template database used for
+		 * creating this database may have been dropped, as noted above.
+		 * Moving a database from one tablespace may also be a partner in the
+		 * crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogRememberMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping replay of database creation WAL record"),
+					 errdetail("The source database directory \"%s\" was not found.",
+							   src_path),
+					 errhint("A future WAL record that removes the directory before reaching consistent mode is expected.")));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2563,6 +2616,10 @@ dbase_redo(XLogReaderState *record)
 				ereport(WARNING,
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
+
+			if (!reachedConsistency)
+				XLogForgetMissingDir(xlrec->tablespace_ids[i], xlrec->db_id);
+
 			pfree(dst_path);
 		}
 
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..55f40831da 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -57,6 +57,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -1574,6 +1575,22 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		/*
+		 * Before we remove the tablespace directory, update minimum recovery
+		 * point to cover this WAL record. Once the tablespace is removed,
+		 * there's no going back.  This manually enforces the WAL-first rule.
+		 * Doing this before the removal means that if the removal fails for
+		 * some reason, the directory is left alone and needs to be manually
+		 * removed.  Alternatively we could update the minimum recovery point
+		 * after removal, but that would leave a small window where the
+		 * WAL-first rule could be violated.
+		 */
+		if (!reachedConsistency)
+			XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..8d48f003b0 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -65,6 +65,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogRememberMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
diff --git a/src/test/recovery/t/029_replay_tsp_drops.pl b/src/test/recovery/t/029_replay_tsp_drops.pl
new file mode 100644
index 0000000000..90a72be489
--- /dev/null
+++ b/src/test/recovery/t/029_replay_tsp_drops.pl
@@ -0,0 +1,67 @@
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test recovery involving tablespace removal.  If recovery stops
+# after once tablespace is removed, the next recovery should properly
+# ignore the operations within the removed tablespaces.
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary1');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+$node_primary->psql('postgres',
+qq[
+	SET allow_in_place_tablespaces=on;
+	CREATE TABLESPACE dropme_ts1 LOCATION '';
+	CREATE TABLESPACE dropme_ts2 LOCATION '';
+	CREATE TABLESPACE source_ts  LOCATION '';
+	CREATE TABLESPACE target_ts  LOCATION '';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = PostgreSQL::Test::Cluster->new('standby1');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREATE DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start, 1, "standby started successfully");
+
+my $log = PostgreSQL::Test::Utils::slurp_file($node_standby->logfile);
+like(
+	$log,
+	qr[WARNING:  skipping replay of database creation WAL record],
+	"warning message is logged");
+
+done_testing();
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 93d5190508..4d58159b18 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3736,6 +3736,8 @@ xl_invalid_page
 xl_invalid_page_key
 xl_invalidations
 xl_logical_message
+xl_missing_dir_key
+xl_missing_dir
 xl_multi_insert_tuple
 xl_multixact_create
 xl_multixact_truncate
-- 
2.30.2

From 3d5b24691517c1aac4b49728abb122c66a4e33be Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Mon, 28 Mar 2022 16:29:04 +0900
Subject: [PATCH 1/2] Tentative test for tsp replay fix

---
 src/test/perl/PostgresNode.pm             | 342 +++++++++++++++++++++-
 src/test/recovery/t/011_crash_recovery.pl | 108 ++++++-
 2 files changed, 447 insertions(+), 3 deletions(-)

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 7b2ec29bb7..88fa08b61d 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -104,6 +104,8 @@ use TestLib ();
 use Time::HiRes qw(usleep);
 use Scalar::Util qw(blessed);
 
+my $windows_os = 0;
+
 our @EXPORT = qw(
   get_new_node
   get_free_port
@@ -323,6 +325,64 @@ sub archive_dir
 
 =pod
 
+=item $node->tablespace_storage([, nocreate])
+
+Diretory to store tablespace directories.
+If nocreate is true, returns undef if not yet created.
+
+=cut
+
+sub tablespace_storage
+{
+	my ($self, $nocreate) = @_;
+
+	if (!defined $self->{_tsproot})
+	{
+		# tablespace is not used, return undef if nocreate is specified.
+		return undef if ($nocreate);
+
+		# create and remember the tablespae root directotry.
+		$self->{_tsproot} = TestLib::tempdir_short();
+	}
+
+	return $self->{_tsproot};
+}
+
+=pod
+
+=item $node->tablespaces()
+
+Returns a hash from tablespace OID to tablespace directory name.  For
+example, an oid 16384 pointing to /tmp/jWAhkT_fs0/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub tablespaces
+{
+	my ($self) = @_;
+	my $pg_tblspc = $self->data_dir . '/pg_tblspc';
+	my %ret;
+
+	# return undef if no tablespace is used
+	return undef if (!defined $self->tablespace_storage(1));
+
+	# collect tablespace entries in pg_tblspc directory
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return %ret;
+}
+
+=pod
+
 =item $node->backup_dir()
 
 The output path for backups taken with $node->backup()
@@ -338,6 +398,77 @@ sub backup_dir
 
 =pod
 
+=item $node->backup_tablespace_storage_path(backup_name)
+
+Returns tablespace location path for backup_name.
+Retuns the parent directory if backup_name is not given.
+
+=cut
+
+sub backup_tablespace_storage_path
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_dir . '/__tsps';
+
+	$dir .= "/$backup_name" if (defined $backup_name);
+
+	return $dir;
+}
+
+=pod
+
+=item $node->backup_create_tablespace_storage(backup_name)
+
+Create tablespace location directory for backup_name if not yet.
+Create the parent tablespace storage that holds all location
+directories if backup_name is not supplied.
+
+=cut
+
+sub backup_create_tablespace_storage
+{
+	my ($self, $backup_name) = @_;
+	my $dir = $self->backup_tablespace_storage_path($backup_name);
+
+	File::Path::make_path $dir if (! -d $dir);
+}
+
+=pod
+
+=item $node->backup_tablespaces(backup_name)
+
+Returns a reference to hash from tablespace OID to tablespace
+directory name of tablespace directory that the specified backup has.
+For example, an oid 16384 pointing to ../tsps/backup1/ts1 is stored as
+$hash{16384} = "ts1".
+
+=cut
+
+sub backup_tablespaces
+{
+	my ($self, $backup_name) = @_;
+	my $pg_tblspc = $self->backup_dir . '/' . $backup_name . '/pg_tblspc';
+	my %ret;
+
+	#return undef if this backup holds no tablespaces
+	return undef if (! -d $self->backup_tablespace_storage_path($backup_name));
+
+	# scan pg_tblspc directory of the backup
+	opendir(my $dir, $pg_tblspc);
+	while (my $oid = readdir($dir))
+	{
+		next if ($oid !~ /^([0-9]+)$/);
+		my $linkpath = "$pg_tblspc/$oid";
+		my $tsppath = dir_readlink($linkpath);
+		$ret{$oid} = File::Basename::basename($tsppath);
+	}
+	closedir($dir);
+
+	return \%ret;
+}
+
+=pod
+
 =item $node->info()
 
 Return a string containing human-readable diagnostic information (paths, etc)
@@ -354,6 +485,7 @@ sub info
 	print $fh "Data directory: " . $self->data_dir . "\n";
 	print $fh "Backup directory: " . $self->backup_dir . "\n";
 	print $fh "Archive directory: " . $self->archive_dir . "\n";
+	print $fh "Tablespace directory: " . $self->tablespace_storage . "\n";
 	print $fh "Connection string: " . $self->connstr . "\n";
 	print $fh "Log file: " . $self->logfile . "\n";
 	close $fh or die;
@@ -536,6 +668,43 @@ sub append_conf
 
 =pod
 
+=item $node->new_tablespace(name)
+
+Create a tablespace directory with the name then returns the path.
+
+=cut
+
+sub new_tablespace
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+
+	die "tablespace \"$name\" already exists" if (!mkdir($path));
+
+	return $path;
+}
+
+=pod
+
+=item $node->tablespace_dir(name)
+
+Return the path of the existing tablespace with the name.
+
+=cut
+
+sub tablespace_dir
+{
+	my ($self, $name) = @_;
+
+	my $path = $self->tablespace_storage . '/' . $name;
+	return undef if (!-d $path);
+
+	return $path;
+}
+
+=pod
+
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
@@ -555,13 +724,54 @@ sub backup
 	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
+	my @tsp_maps;
+
+	# Build tablespace mappings.  We once let pg_basebackup copy
+	# tablespaces into temporary tablespace storage with a short name
+	# so that we can work on pathnames that fit our tar format which
+	# pg_basebackup depends on.
+	my $map_src_root = $self->tablespace_storage(1);
+	my $backup_tmptsp_root = TestLib::tempdir_short();
+	my %tsps = $self->tablespaces();
+	foreach my $tspname (values %tsps)
+	{
+		my $src = "$map_src_root/$tspname";
+		my $dst = "$backup_tmptsp_root/$tspname";
+		push(@tsp_maps, "--tablespace-mapping=$src=$dst");
+	}
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail(
 		'pg_basebackup', '-D', $backup_path, '-h',
 		$self->host,     '-p', $self->port,  '--checkpoint',
 		'fast',          '--no-sync',
+		@tsp_maps,
 		@{ $params{backup_options} });
+
+	# Move the tablespaces from temporary storage into backup
+	# directory, unless the backup is in tar mode.
+	if (%tsps && ! -f "$backup_path/base.tar")
+	{
+		$self->backup_create_tablespace_storage();
+		RecursiveCopy::copypath(
+			$backup_tmptsp_root,
+			$self->backup_tablespace_storage_path($backup_name));
+		# delete the temporary directory right away
+		rmtree $backup_tmptsp_root;
+
+		# Fix tablespace symlinks.  This is not necessarily required
+		# in backups but keep them consistent.
+		my $linkdst_root = "$backup_path/pg_tblspc";
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			unlink $tspdst;
+			dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	print "# Backup finished\n";
 	return;
 }
@@ -623,11 +833,32 @@ sub _backup_fs
 	RecursiveCopy::copypath(
 		$self->data_dir,
 		$backup_path,
+		# Skipping some files and tablespace symlinks
 		filterfn => sub {
 			my $src = shift;
-			return ($src ne 'log' and $src ne 'postmaster.pid');
+			return ($src ne 'log' and $src ne 'postmaster.pid' and
+					$src !~ m!^pg_tblspc/[0-9]+$!);
 		});
 
+	# Copy tablespaces if any
+	my %tsps = $self->tablespaces();
+	if (%tsps)
+	{
+		$self->backup_create_tablespace_storage();
+		RecursiveCopy::copypath(
+			$self->tablespace_storage,
+			$self->backup_tablespace_storage_path($backup_name));
+
+		my $linkdst_root = $backup_path . '/pg_tblspc';
+		my $linksrc_root = $self->backup_tablespace_storage_path($backup_name);
+		foreach my $oid (keys %tsps)
+		{
+			my $tspdst = "$linkdst_root/$oid";
+			my $tspsrc = "$linksrc_root/" . $tsps{$oid};
+			dir_symlink($tspsrc, $tspdst);
+		}
+	}
+
 	if ($hot)
 	{
 
@@ -645,6 +876,80 @@ sub _backup_fs
 
 
 
+=pod
+
+=item dir_symlink(oldname, newname)
+
+Portably create a symlink for a directory. On Windows this creates a junction
+point. Elsewhere it just calls perl's builtin symlink.
+
+=cut
+
+sub dir_symlink
+{
+	my $oldname = shift;
+	my $newname = shift;
+	if ($windows_os)
+	{
+		$oldname =~ s,/,\\,g;
+		$newname =~ s,/,\\,g;
+		my $cmd = qq{mklink /j "$newname" "$oldname"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+		system($cmd);
+	}
+	else
+	{
+		symlink $oldname, $newname;
+	}
+	die "No $newname" unless -e $newname;
+}
+
+=pod
+
+=item dir_readlink(name)
+
+Portably read a symlink for a directory. On Windows this reads a junction
+point. Elsewhere it just calls perl's builtin readlink.
+
+=cut
+
+sub dir_readlink
+{
+	my $name = shift;
+	if ($windows_os)
+	{
+		$name .= '/..';
+		$name =~ s,/,\\,g;
+		# Split the path into parent directory and link name
+		die "invalid path spec: $name" if ($name !~ m!^(.*)\\([^\\]+)\\?$!);
+		my ($dir, $fname) = ($1, $2);
+		my $cmd = qq{cmd /c "dir /A:L $dir"};
+		if ($Config{osname} eq 'msys')
+		{
+			# need some indirection on msys
+			$cmd = qq{echo '$cmd' | \$COMSPEC /Q};
+		}
+
+		my $result;
+		foreach my $l (split /[\r\n]+/, `$cmd`)
+		{
+			$result = $1 if ($l =~ m/<JUNCTION>\W+$fname \[(.*)\]/)
+		}
+		die "junction $name not found" if (!defined $result);
+
+		$name =~ s,\\,/,g;
+		return $result;
+	}
+	else
+	{
+		return readlink $name;
+	}
+}
+
 =pod
 
 =item $node->init_from_backup(root_node, backup_name)
@@ -689,7 +994,40 @@ sub init_from_backup
 
 	my $data_path = $self->data_dir;
 	rmdir($data_path);
-	RecursiveCopy::copypath($backup_path, $data_path);
+
+	RecursiveCopy::copypath(
+		$backup_path,
+		$data_path,
+		# Skipping tablespace symlinks
+		filterfn => sub {
+			my $src = shift;
+			return ($src !~ m!^pg_tblspc/[0-9]+$!);
+		});
+
+	# Copy tablespaces if any
+	my $tsps = $root_node->backup_tablespaces($backup_name);
+
+	if ($tsps)
+	{
+		my $tsp_src = $root_node->backup_tablespace_storage_path($backup_name);
+		my $tsp_dst = $self->tablespace_storage();
+		my $linksrc_root = $data_path . '/pg_tblspc';
+
+		# copypath() rejects to copy into existing directory.
+		# Copy individual directories in the storage.
+		foreach my $oid (keys %{$tsps})
+		{
+			my $tsp = ${$tsps}{$oid};
+			my $tspsrc = "$tsp_src/$tsp";
+			my $tspdst = "$tsp_dst/$tsp";
+			RecursiveCopy::copypath($tspsrc, $tspdst);
+
+			# Create tablespace symlink for this tablespace
+			my $linkdst = "$linksrc_root/$oid";
+			dir_symlink($tspdst, $linkdst);
+		}
+	}
+
 	chmod(0700, $data_path);
 
 	# Base configuration for this node
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 5dc52412ca..30aaf763e5 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -15,7 +15,7 @@ if ($Config{osname} eq 'MSWin32')
 }
 else
 {
-	plan tests => 3;
+	plan tests => 5;
 }
 
 my $node = get_new_node('master');
@@ -66,3 +66,109 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
 	'aborted', 'xid is aborted after crash');
 
 $tx->kill_kill;
+
+my $node_primary = get_new_node('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+my $dropme_ts_primary1 = $node_primary->new_tablespace('dropme_ts1');
+my $dropme_ts_primary2 = $node_primary->new_tablespace('dropme_ts2');
+my $soruce_ts_primary = $node_primary->new_tablespace('source_ts');
+my $target_ts_primary = $node_primary->new_tablespace('target_ts');
+
+$node_primary->psql('postgres',
+qq[
+	CREATE TABLESPACE dropme_ts1 LOCATION '$dropme_ts_primary1';
+	CREATE TABLESPACE dropme_ts2 LOCATION '$dropme_ts_primary2';
+	CREATE TABLESPACE source_ts  LOCATION '$soruce_ts_primary';
+	CREATE TABLESPACE target_ts  LOCATION '$target_ts_primary';
+    CREATE DATABASE template_db IS_TEMPLATE = true;
+]);
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+
+my $node_standby = get_new_node('standby2');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure connection is made
+$node_primary->poll_query_until(
+	'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+# to be applied to already-removed directories.
+$node_primary->safe_psql('postgres',
+						q[CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
+						  CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
+						  CREATE DATABASE moveme_db TABLESPACE source_ts;
+						  ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+						  CREATE DATABASE newdb TEMPLATE template_db;
+						  ALTER DATABASE template_db IS_TEMPLATE = false;
+						  DROP DATABASE dropme_db1;
+						  DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+						  DROP TABLESPACE source_ts;
+						  DROP DATABASE template_db;]);
+
+$node_primary->wait_for_catchup($node_standby, 'replay',
+							   $node_primary->lsn('replay'));
+$node_standby->stop('immediate');
+
+# Should restart ignoring directory creation error.
+is($node_standby->start(fail_ok => 1), 1);
+
+
+# TEST 5
+#
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).
+
+$node_primary = get_new_node('primary3');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+my $ts_primary = $node_primary->new_tablespace('dropme_ts1');
+$node_primary->safe_psql('postgres',
+						 "CREATE TABLESPACE ts1 LOCATION '$ts_primary'");
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 TABLESPACE ts1");
+
+# Take backup
+$backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+$node_standby = get_new_node('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+File::Path::rmtree($node_standby->tablespace_dir('dropme_ts1'));
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+if ($node_primary->poll_query_until(
+		'postgres',
+		'SELECT count(*) = 0 FROM pg_stat_replication',
+		't') == 1)
+{
+	pass('standby failed as expected');
+	# We know that the standby has failed.  Setting its pid to
+	# undefined avoids error when PostgreNode module tries to stop the
+	# standby node as part of tear_down sequence.
+	$node_standby->{_pid} = undef;
+}
+else
+{
+	fail('standby did not fail within 5 seconds');
+}
+
-- 
2.27.0

From bfd70d2ab7aaf5b5791c46d78e6bf087041abb0f Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Mon, 28 Mar 2022 16:29:33 +0900
Subject: [PATCH 2/2] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the standby
would fail to recover in such a case.  However, the directories could be
legitimately missing.  Consider a sequence of WAL records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch adds a mechanism similar to invalid-page tracking, to keep a
tally of missing directories during crash recovery.  If all the missing
directory references are matched with corresponding drop records at the
end of crash recovery, the standby can safely continue following the
primary.

Backpatch to from 10 to 12. This fix has already been committed to 13
and later.

A new TAP test file is added to verify the condition.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 src/backend/access/transam/xlog.c      |   6 +
 src/backend/access/transam/xlogutils.c | 159 ++++++++++++++++++++++++-
 src/backend/commands/dbcommands.c      |  55 +++++++++
 src/backend/commands/tablespace.c      |  17 +++
 src/include/access/xlogutils.h         |   4 +
 src/tools/pgindent/typedefs.list       |   2 +
 6 files changed, 242 insertions(+), 1 deletion(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7141e5dca8..3d3342b714 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8006,6 +8006,12 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check if the XLOG sequence contained any unresolved references to
+		 * missing directories.
+		 */
+		XLogCheckMissingDirs();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..11c40b7446 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -31,6 +31,164 @@
 #include "utils/rel.h"
 
 
+
+/*
+ * If a create database WAL record is being replayed more than once during
+ * crash recovery on a standby, it is possible that either the tablespace
+ * directory or the template database directory is missing.  This happens when
+ * the directories are removed by replay of subsequent drop records.  Note
+ * that this problem happens only on standby and not on master.  On master, a
+ * checkpoint is created at the end of create database operation. On standby,
+ * however, such a strategy (creating restart points during replay) is not
+ * viable because it will slow down WAL replay.
+ *
+ * The alternative is to track references to each missing directory
+ * encountered when performing crash recovery in the following hash table.
+ * Similar to invalid page table above, the expectation is that each missing
+ * directory entry should be matched with a drop database or drop tablespace
+ * WAL record by the end of crash recovery.
+ */
+typedef struct xl_missing_dir_key
+{
+	Oid			spcNode;
+	Oid			dbNode;
+} xl_missing_dir_key;
+
+typedef struct xl_missing_dir
+{
+	xl_missing_dir_key key;
+	char		path[MAXPGPATH];
+} xl_missing_dir;
+
+static HTAB *missing_dir_tab = NULL;
+
+
+/*
+ * Keep track of a directory that wasn't found while replaying database
+ * creation records.  These should match up with tablespace removal records
+ * later in the WAL stream; we verify that before reaching consistency.
+ */
+void
+XLogRememberMissingDir(Oid spcNode, Oid dbNode, char *path)
+{
+	xl_missing_dir_key key;
+	bool		found;
+	xl_missing_dir *entry;
+
+	/*
+	 * Database OID may be invalid but tablespace OID must be valid.  If
+	 * dbNode is InvalidOid, we are logging a missing tablespace directory,
+	 * otherwise we are logging a missing database directory.
+	 */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+	{
+		/* create hash table when first needed */
+		HASHCTL		ctl;
+
+		memset(&ctl, 0, sizeof(ctl));
+		ctl.keysize = sizeof(xl_missing_dir_key);
+		ctl.entrysize = sizeof(xl_missing_dir);
+
+		missing_dir_tab = hash_create("XLOG missing directory table",
+									  100,
+									  &ctl,
+									  HASH_ELEM | HASH_BLOBS);
+	}
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	entry = hash_search(missing_dir_tab, &key, HASH_ENTER, &found);
+
+	if (found)
+	{
+		if (dbNode == InvalidOid)
+			elog(DEBUG1, "missing directory %s (tablespace %u) already exists: %s",
+				 path, spcNode, entry->path);
+		else
+			elog(DEBUG1, "missing directory %s (tablespace %u database %u) already exists: %s",
+				 path, spcNode, dbNode, entry->path);
+	}
+	else
+	{
+		strlcpy(entry->path, path, sizeof(entry->path));
+		if (dbNode == InvalidOid)
+			elog(DEBUG1, "logged missing dir %s (tablespace %u)",
+				 path, spcNode);
+		else
+			elog(DEBUG1, "logged missing dir %s (tablespace %u database %u)",
+				 path, spcNode, dbNode);
+	}
+}
+
+/*
+ * Remove an entry from the list of directories not found.  This is to be done
+ * when the matching tablespace removal WAL record is found.
+ */
+void
+XLogForgetMissingDir(Oid spcNode, Oid dbNode)
+{
+	xl_missing_dir_key key;
+
+	key.spcNode = spcNode;
+	key.dbNode = dbNode;
+
+	/* Database OID may be invalid but tablespace OID must be valid. */
+	Assert(OidIsValid(spcNode));
+
+	if (missing_dir_tab == NULL)
+		return;
+
+	if (hash_search(missing_dir_tab, &key, HASH_REMOVE, NULL) != NULL)
+	{
+		if (dbNode == InvalidOid)
+		{
+			elog(DEBUG2, "forgot missing dir (tablespace %u)", spcNode);
+		}
+		else
+		{
+			char	   *path = GetDatabasePath(dbNode, spcNode);
+
+			elog(DEBUG2, "forgot missing dir %s (tablespace %u database %u)",
+				 path, spcNode, dbNode);
+			pfree(path);
+		}
+	}
+}
+
+/*
+ * This is called at the end of crash recovery, before entering archive
+ * recovery on a standby.  PANIC if the hash table is not empty.
+ */
+void
+XLogCheckMissingDirs(void)
+{
+	HASH_SEQ_STATUS status;
+	xl_missing_dir *hentry;
+	bool		foundone = false;
+
+	if (missing_dir_tab == NULL)
+		return;					/* nothing to do */
+
+	hash_seq_init(&status, missing_dir_tab);
+
+	while ((hentry = (xl_missing_dir *) hash_seq_search(&status)) != NULL)
+	{
+		elog(WARNING, "missing directory \"%s\" tablespace %u database %u",
+			 hentry->path, hentry->key.spcNode, hentry->key.dbNode);
+		foundone = true;
+	}
+
+	if (foundone)
+		elog(PANIC, "WAL contains references to missing directories");
+
+	hash_destroy(missing_dir_tab);
+	missing_dir_tab = NULL;
+}
+
+
 /*
  * During XLOG replay, we may see XLOG records for incremental updates of
  * pages that no longer exist, because their relation was later dropped or
@@ -56,7 +214,6 @@ typedef struct xl_invalid_page
 
 static HTAB *invalid_page_tab = NULL;
 
-
 /* Report a reference to an invalid page */
 static void
 report_invalid_page(int elevel, RelFileNode node, ForkNumber forkno,
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 863f89f19d..44512a8a30 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -2108,7 +2108,9 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
+		bool		skip = false;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
 		dst_path = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
@@ -2126,6 +2128,56 @@ dbase_redo(XLogReaderState *record)
 						(errmsg("some useless files may be left behind in old database directory \"%s\"",
 								dst_path)));
 		}
+		else if (!reachedConsistency)
+		{
+			/*
+			 * It is possible that a drop tablespace record appearing later in
+			 * WAL has already been replayed -- in other words, that we are
+			 * replaying the database creation record a second time with no
+			 * intervening checkpoint.  In that case, the tablespace directory
+			 * has already been removed and the create database operation
+			 * cannot be replayed.  Skip the replay itself, but remember the
+			 * fact that the tablespace directory is missing, to be matched
+			 * with the expected tablespace drop record later.
+			 */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			if (!(stat(parent_path, &st) == 0 && S_ISDIR(st.st_mode)))
+			{
+				XLogRememberMissingDir(xlrec->tablespace_id, InvalidOid, parent_path);
+				skip = true;
+				ereport(WARNING,
+						(errmsg("skipping replay of database creation WAL record"),
+						 errdetail("The target tablespace \"%s\" directory was not found.",
+								   parent_path),
+						 errhint("A future WAL record that removes the directory before reaching consistent mode is expected.")));
+			}
+			pfree(parent_path);
+		}
+
+		/*
+		 * If the source directory is missing, skip the copy and make a note of
+		 * it for later.
+		 *
+		 * One possible reason for this is that the template database used for
+		 * creating this database may have been dropped, as noted above.
+		 * Moving a database from one tablespace may also be a partner in the
+		 * crime.
+		 */
+		if (!(stat(src_path, &st) == 0 && S_ISDIR(st.st_mode)) &&
+			!reachedConsistency)
+		{
+			XLogRememberMissingDir(xlrec->src_tablespace_id, xlrec->src_db_id, src_path);
+			skip = true;
+			ereport(WARNING,
+					(errmsg("skipping replay of database creation WAL record"),
+					 errdetail("The source database directory \"%s\" was not found.",
+							   src_path),
+					 errhint("A future WAL record that removes the directory before reaching consistent mode is expected.")));
+		}
+
+		if (skip)
+			return;
 
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
@@ -2181,6 +2233,9 @@ dbase_redo(XLogReaderState *record)
 					(errmsg("some useless files may be left behind in old database directory \"%s\"",
 							dst_path)));
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->tablespace_id, xlrec->db_id);
+
 		if (InHotStandby)
 		{
 			/*
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index f060c24599..5b600a98ff 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -58,6 +58,7 @@
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"
@@ -1530,6 +1531,22 @@ tblspc_redo(XLogReaderState *record)
 	{
 		xl_tblspc_drop_rec *xlrec = (xl_tblspc_drop_rec *) XLogRecGetData(record);
 
+		if (!reachedConsistency)
+			XLogForgetMissingDir(xlrec->ts_id, InvalidOid);
+
+		/*
+		 * Before we remove the tablespace directory, update minimum recovery
+		 * point to cover this WAL record. Once the tablespace is removed,
+		 * there's no going back.  This manually enforces the WAL-first rule.
+		 * Doing this before the removal means that if the removal fails for
+		 * some reason, the directory is left alone and needs to be manually
+		 * removed.  Alternatively we could update the minimum recovery point
+		 * after removal, but that would leave a small window where the
+		 * WAL-first rule could be violated.
+		 */
+		if (!reachedConsistency)
+			XLogFlush(record->EndRecPtr);
+
 		/*
 		 * If we issued a WAL record for a drop tablespace it implies that
 		 * there were no files in it at all when the DROP was done. That means
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..a17c204638 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -23,6 +23,10 @@ extern void XLogDropDatabase(Oid dbid);
 extern void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
 								 BlockNumber nblocks);
 
+extern void XLogRememberMissingDir(Oid spcNode, Oid dbNode, char *path);
+extern void XLogForgetMissingDir(Oid spcNode, Oid dbNode);
+extern void XLogCheckMissingDirs(void);
+
 /* Result codes for XLogReadBufferForRedo[Extended] */
 typedef enum
 {
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index daebb77387..bdf6b25d59 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -3404,6 +3404,8 @@ xl_invalid_page
 xl_invalid_page_key
 xl_invalidations
 xl_logical_message
+xl_missing_dir_key
+xl_missing_dir
 xl_multi_insert_tuple
 xl_multixact_create
 xl_multixact_truncate
-- 
2.27.0

From c4107b4953d251f7fab06400d2c2ae2e0a505759 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Tue, 5 Apr 2022 15:31:45 +0900
Subject: [PATCH v22] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch allows missing tablespaces to be created during recovery
before reaching consistency.  The tablespaces are created as real
directories that should not exists but will be removed until reaching
consistency. CheckRecoveryConsistency is responsible to make sure they
have disappeared.

Similar to log_invalid_page mechanism, the GUC ignore_invalid_pages
turns into PANIC errors detected by this patch into WARNING, which
allows continueing recovery.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 doc/src/sgml/config.sgml                    |   5 +-
 src/backend/access/transam/xlogrecovery.c   |  53 +++++++
 src/backend/commands/dbcommands.c           |  71 +++++++++
 src/backend/commands/tablespace.c           |  28 +---
 src/backend/utils/misc/guc.c                |   8 +-
 src/include/access/xlogutils.h              |   2 +
 src/test/recovery/t/029_replay_tsp_drops.pl | 155 ++++++++++++++++++++
 7 files changed, 290 insertions(+), 32 deletions(-)
 create mode 100644 src/test/recovery/t/029_replay_tsp_drops.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 43e4ade83e..c22229468b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -11228,11 +11228,12 @@ LOG:  CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1)
       <listitem>
        <para>
         If set to <literal>off</literal> (the default), detection of
-        WAL records having references to invalid pages during
+        WAL records having references to invalid pages or
+        WAL records resulting in invalid directory operations during
         recovery causes <productname>PostgreSQL</productname> to
         raise a PANIC-level error, aborting the recovery. Setting
         <varname>ignore_invalid_pages</varname> to <literal>on</literal>
-        causes the system to ignore invalid page references in WAL records
+        causes the system to ignore invalid actions caused by such WAL records
         (but still report a warning), and continue the recovery.
         This behavior may <emphasis>cause crashes, data loss,
         propagate or hide corruption, or other serious problems</emphasis>.
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 8d2395dae2..0889ba4b47 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -1986,6 +1986,51 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Makes sure that ./pg_tblspc directory doesn't contain a real directory.
+ *
+ * This is intended to be called after reaching consistency.
+ * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
+ * recovery can continue.
+ * 
+ * Note that it is the normal behavior when allow_in_place_tablespaces=on, but
+ * we don't bother caring that case since it is a developer-only setting.
+ */
+static void
+CheckTablespaceDirectory(void)
+{
+	char *tblspc_path = "./pg_tblspc";
+	DIR		   *dir;
+	struct dirent *de;
+
+	dir = AllocateDir(tblspc_path);
+	while ((de = ReadDir(dir, tblspc_path)) != NULL)
+	{
+		char	path[MAXPGPATH];
+		char   *p;
+		struct stat st;
+
+		/* Skip entries of non-oid names */
+		for (p = de->d_name; *p && isdigit(*p); p++);
+		if (*p)
+			continue;
+
+		snprintf(path, MAXPGPATH, "%s/%s", tblspc_path, de->d_name);
+
+#ifndef WIN32
+		if (lstat(path, &st) < 0)
+			ereport(ERROR, errcode_for_file_access(),
+					errmsg("could not stat file \"%s\": %m", path));
+
+		if (!S_ISLNK(st.st_mode))
+#else
+		if (!pgwin32_is_junction(path))
+#endif
+			elog(ignore_invalid_pages ? WARNING : PANIC,
+				 "real directory found in pg_tblspc directory: %s", de->d_name);
+	}
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
@@ -2051,6 +2096,14 @@ CheckRecoveryConsistency(void)
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
 						LSN_FORMAT_ARGS(lastReplayedEndRecPtr))));
+
+		/*
+		 * Check that pg_tblspc doesn't contain a real
+		 * directory. Database/CREATE_* records may create a tablespace
+		 * directory that should have been removed until consistency is
+		 * reached.
+		 */
+		CheckTablespaceDirectory();
 	}
 
 	/*
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index df16533901..910101da01 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -47,6 +48,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -62,6 +64,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/guc.h"
 #include "utils/pg_locale.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -135,6 +138,7 @@ static void CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid,
 									bool isRedo);
 static void CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dboid, Oid src_tsid,
 										Oid dst_tsid);
+static void maybe_create_directory(char *path);
 
 /*
  * Create a new database using the WAL_LOG strategy.
@@ -3003,6 +3007,43 @@ get_database_name(Oid dbid)
 	return result;
 }
 
+/*
+ * maybe_create_directory()
+ *
+ * During recovery, there's a case where we validly need to recover a missing
+ * tablespace directory so that recovery can continue.  This happens when
+ * recovery wants to create a database but the holding tablespace has been
+ * removed before the server stopped.  Since we expect that the directory will
+ * be gone before reaching recovery consistency, and we have no knowledge about
+ * the tablespace other than its OID here, we create a real directory under
+ * pg_tblspc here instead of restoring the symlink.  ignore_invalid_pages=on
+ * reduces the error level so that recovery can continue.
+ */
+static void
+maybe_create_directory(char *path)
+{
+	struct stat	st;
+
+	Assert(RecoveryInProgress());
+
+	if (stat(path, &st) == 0)
+		return;
+
+	/* XXX: Do we make sure that the path is under pg_tblspc? */
+
+	if (reachedConsistency && !ignore_invalid_pages)
+		ereport(PANIC,
+				errmsg("missing directory \"%s\"", path));
+
+	elog(reachedConsistency ? WARNING : DEBUG1,
+		 "creating missing directory: %s", path);
+
+	if (pg_mkdir_p(path, pg_dir_create_mode) != 0)
+		ereport(PANIC,
+				errmsg("could not create missing directory \"%s\": %m", path));
+}
+
+
 /*
  * DATABASE resource manager's routines
  */
@@ -3039,6 +3080,30 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		if (stat(dst_path, &st) < 0)
+		{
+			char *parent_path;
+
+			if (errno != ENOENT)
+				ereport(FATAL,
+						errmsg("could not stat directory \"%s\": %m",
+							   dst_path));
+
+			/* create the parent directory if needed and valid */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			maybe_create_directory(parent_path);
+		}
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+			maybe_create_directory(src_path);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
@@ -3057,9 +3122,15 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_wal_log_rec *xlrec =
 		(xl_dbase_create_wal_log_rec *) XLogRecGetData(record);
 		char	   *dbpath;
+		char	   *parent_path;
 
 		dbpath = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
 
+		/* create the parent directory if needed and valid */
+		parent_path = pstrdup(dbpath);
+		get_parent_directory(parent_path);
+		maybe_create_directory(parent_path);
+
 		/* Create the database directory with the version file. */
 		CreateDirAndVersionFile(dbpath, xlrec->db_id, xlrec->tablespace_id,
 								true);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..675f578dfe 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -155,8 +155,6 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -169,32 +167,8 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 9e8ab1420d..9134a73d3d 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -42,6 +42,7 @@
 #include "access/xact.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
+#include "access/xlogutils.h"
 #include "catalog/namespace.h"
 #include "catalog/objectaccess.h"
 #include "catalog/pg_authid.h"
@@ -139,7 +140,6 @@ extern int	CommitSiblings;
 extern char *default_tablespace;
 extern char *temp_tablespaces;
 extern bool ignore_checksum_failure;
-extern bool ignore_invalid_pages;
 extern bool synchronize_seqscans;
 
 #ifdef TRACE_SYNCSCAN
@@ -1304,10 +1304,12 @@ static struct config_bool ConfigureNamesBool[] =
 		{"ignore_invalid_pages", PGC_POSTMASTER, DEVELOPER_OPTIONS,
 			gettext_noop("Continues recovery after an invalid pages failure."),
 			gettext_noop("Detection of WAL records having references to "
-						 "invalid pages during recovery causes PostgreSQL to "
+						 "invalid pages or WAL records resulting in invalid "
+						 "directory operations during "
+						 "recovery that cause PostgreSQL"
 						 "raise a PANIC-level error, aborting the recovery. "
 						 "Setting ignore_invalid_pages to true causes "
-						 "the system to ignore invalid page references "
+						 "the system to ignore those inconsistencies "
 						 "in WAL records (but still report a warning), "
 						 "and continue recovery. This behavior may cause "
 						 "crashes, data loss, propagate or hide corruption, "
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..d88661997f 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -54,6 +54,8 @@ typedef enum
 
 extern HotStandbyState standbyState;
 
+extern bool ignore_invalid_pages;
+
 #define InHotStandby (standbyState >= STANDBY_SNAPSHOT_PENDING)
 
 
diff --git a/src/test/recovery/t/029_replay_tsp_drops.pl b/src/test/recovery/t/029_replay_tsp_drops.pl
new file mode 100644
index 0000000000..b401ab8072
--- /dev/null
+++ b/src/test/recovery/t/029_replay_tsp_drops.pl
@@ -0,0 +1,155 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+#
+# Tests relating to PostgreSQL crash recovery and redo
+#
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_tablespace
+{
+	my ($strategy) = @_;
+
+	my $node_primary = PostgreSQL::Test::Cluster->new("primary1_$strategy");
+	$node_primary->init(allows_streaming => 1);
+	$node_primary->start;
+	$node_primary->psql('postgres',
+			qq[
+				SET allow_in_place_tablespaces=on;
+				CREATE TABLESPACE dropme_ts1 LOCATION '';
+				CREATE TABLESPACE dropme_ts2 LOCATION '';
+				CREATE TABLESPACE source_ts  LOCATION '';
+				CREATE TABLESPACE target_ts  LOCATION '';
+				CREATE DATABASE template_db IS_TEMPLATE = true;
+			]);
+	my $backup_name = 'my_backup';
+	$node_primary->backup($backup_name);
+
+	my $node_standby = PostgreSQL::Test::Cluster->new("standby2_$strategy");
+	$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+	$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");	
+	$node_standby->start;
+
+	# Make sure connection is made
+	$node_primary->poll_query_until(
+		'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+	$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+	# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+	# to be applied to already-removed directories.
+	my $query = q[
+	CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1 STRATEGY=<STRATEGY>;
+	CREATE TABLE t (a int) TABLESPACE dropme_ts2;
+	CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2 STRATEGY=<STRATEGY>;
+	CREATE DATABASE moveme_db TABLESPACE source_ts STRATEGY=<STRATEGY>;
+	ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+	CREATE DATABASE newdb TEMPLATE template_db STRATEGY=<STRATEGY>;
+	ALTER DATABASE template_db IS_TEMPLATE = false;
+	DROP DATABASE dropme_db1;
+	DROP TABLE t;
+	DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+	DROP TABLESPACE source_ts;
+	DROP DATABASE template_db;];
+
+	$query =~ s/<STRATEGY>/$strategy/g;
+	$node_primary->safe_psql('postgres', $query);
+	$node_primary->wait_for_catchup($node_standby, 'replay',
+									$node_primary->lsn('replay'));
+
+	# show "create missing directory" log message
+	$node_standby->safe_psql('postgres',
+							 "ALTER SYSTEM SET log_min_messages TO debug1;");
+	$node_standby->stop('immediate');
+	# Should restart ignoring directory creation error.
+	is($node_standby->start(fail_ok => 1), 1);
+	$node_standby->stop('immediate');
+}	
+
+test_tablespace("FILE_COPY");
+test_tablespace("WAL_LOG");
+
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).  This is
+# effective only for CREATE DATABASE WITH STRATEGY=FILE_COPY.
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+$node_primary->safe_psql('postgres', q[
+						 SET allow_in_place_tablespaces=on;
+						 CREATE TABLESPACE ts1 LOCATION '']);
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 WITH TABLESPACE ts1 STRATEGY=FILE_COPY");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");	
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+my $tspoid = $node_standby->safe_psql('postgres',
+									  "SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
+my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
+File::Path::rmtree($tspdir);
+
+my $logstart = get_log_size($node_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1 STRATEGY=FILE_COPY;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+# In this test, PANIC turns into WARNING by ignore_invalid_pages.
+# Check the log messages instead of confirming standby failure.
+my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
+while ($max_attempts-- >= 0)
+{
+	last if (find_in_log(
+				 $node_standby,
+				 "WARNING:  creating missing directory: pg_tblspc/",
+				 $logstart));
+	sleep 1;
+}
+ok($max_attempts > 0, "invalid directory creation is detected");
+
+done_testing();
+
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
-- 
2.27.0

From 1e7f5e5e10ea504e168d01b3db8be12c2f63b6d6 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Tue, 5 Apr 2022 15:31:45 +0900
Subject: [PATCH v23] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch allows missing tablespaces to be created during recovery
before reaching consistency.  The tablespaces are created as real
directories that should not exists but will be removed until reaching
consistency. CheckRecoveryConsistency is responsible to make sure they
have disappeared.

Similar to log_invalid_page mechanism, the GUC ignore_invalid_pages
turns into PANIC errors detected by this patch into WARNING, which
allows continueing recovery.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 doc/src/sgml/config.sgml                    |   5 +-
 src/backend/access/transam/xlogrecovery.c   |  55 +++++++
 src/backend/commands/dbcommands.c           |  71 +++++++++
 src/backend/commands/tablespace.c           |  28 +---
 src/backend/utils/misc/guc.c                |   8 +-
 src/include/access/xlogutils.h              |   2 +
 src/test/recovery/t/029_replay_tsp_drops.pl | 155 ++++++++++++++++++++
 7 files changed, 292 insertions(+), 32 deletions(-)
 create mode 100644 src/test/recovery/t/029_replay_tsp_drops.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 43e4ade83e..c22229468b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -11228,11 +11228,12 @@ LOG:  CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1)
       <listitem>
        <para>
         If set to <literal>off</literal> (the default), detection of
-        WAL records having references to invalid pages during
+        WAL records having references to invalid pages or
+        WAL records resulting in invalid directory operations during
         recovery causes <productname>PostgreSQL</productname> to
         raise a PANIC-level error, aborting the recovery. Setting
         <varname>ignore_invalid_pages</varname> to <literal>on</literal>
-        causes the system to ignore invalid page references in WAL records
+        causes the system to ignore invalid actions caused by such WAL records
         (but still report a warning), and continue the recovery.
         This behavior may <emphasis>cause crashes, data loss,
         propagate or hide corruption, or other serious problems</emphasis>.
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 8d2395dae2..18dcc452ca 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -1986,6 +1986,53 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Makes sure that ./pg_tblspc directory doesn't contain a real directory.
+ *
+ * This is intended to be called after reaching consistency.
+ * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
+ * recovery can continue.
+ * 
+ * Note that it is the normal behavior when allow_in_place_tablespaces=on, but
+ * we don't bother caring that case since it is a developer-only setting.
+ */
+static void
+CheckTablespaceDirectory(void)
+{
+	char *tblspc_path = "./pg_tblspc";
+	DIR		   *dir;
+	struct dirent *de;
+
+	dir = AllocateDir(tblspc_path);
+	while ((de = ReadDir(dir, tblspc_path)) != NULL)
+	{
+		char	path[MAXPGPATH];
+		char   *p;
+#ifndef WIN32
+		struct stat st;
+#endif
+
+		/* Skip entries of non-oid names */
+		for (p = de->d_name; *p && isdigit(*p); p++);
+		if (*p)
+			continue;
+
+		snprintf(path, MAXPGPATH, "%s/%s", tblspc_path, de->d_name);
+
+#ifndef WIN32
+		if (lstat(path, &st) < 0)
+			ereport(ERROR, errcode_for_file_access(),
+					errmsg("could not stat file \"%s\": %m", path));
+
+		if (!S_ISLNK(st.st_mode))
+#else
+		if (!pgwin32_is_junction(path))
+#endif
+			elog(ignore_invalid_pages ? WARNING : PANIC,
+				 "real directory found in pg_tblspc directory: %s", de->d_name);
+	}
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
@@ -2051,6 +2098,14 @@ CheckRecoveryConsistency(void)
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
 						LSN_FORMAT_ARGS(lastReplayedEndRecPtr))));
+
+		/*
+		 * Check that pg_tblspc doesn't contain a real
+		 * directory. Database/CREATE_* records may create a tablespace
+		 * directory that should have been removed until consistency is
+		 * reached.
+		 */
+		CheckTablespaceDirectory();
 	}
 
 	/*
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index df16533901..910101da01 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -47,6 +48,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -62,6 +64,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/guc.h"
 #include "utils/pg_locale.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -135,6 +138,7 @@ static void CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid,
 									bool isRedo);
 static void CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dboid, Oid src_tsid,
 										Oid dst_tsid);
+static void maybe_create_directory(char *path);
 
 /*
  * Create a new database using the WAL_LOG strategy.
@@ -3003,6 +3007,43 @@ get_database_name(Oid dbid)
 	return result;
 }
 
+/*
+ * maybe_create_directory()
+ *
+ * During recovery, there's a case where we validly need to recover a missing
+ * tablespace directory so that recovery can continue.  This happens when
+ * recovery wants to create a database but the holding tablespace has been
+ * removed before the server stopped.  Since we expect that the directory will
+ * be gone before reaching recovery consistency, and we have no knowledge about
+ * the tablespace other than its OID here, we create a real directory under
+ * pg_tblspc here instead of restoring the symlink.  ignore_invalid_pages=on
+ * reduces the error level so that recovery can continue.
+ */
+static void
+maybe_create_directory(char *path)
+{
+	struct stat	st;
+
+	Assert(RecoveryInProgress());
+
+	if (stat(path, &st) == 0)
+		return;
+
+	/* XXX: Do we make sure that the path is under pg_tblspc? */
+
+	if (reachedConsistency && !ignore_invalid_pages)
+		ereport(PANIC,
+				errmsg("missing directory \"%s\"", path));
+
+	elog(reachedConsistency ? WARNING : DEBUG1,
+		 "creating missing directory: %s", path);
+
+	if (pg_mkdir_p(path, pg_dir_create_mode) != 0)
+		ereport(PANIC,
+				errmsg("could not create missing directory \"%s\": %m", path));
+}
+
+
 /*
  * DATABASE resource manager's routines
  */
@@ -3039,6 +3080,30 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		if (stat(dst_path, &st) < 0)
+		{
+			char *parent_path;
+
+			if (errno != ENOENT)
+				ereport(FATAL,
+						errmsg("could not stat directory \"%s\": %m",
+							   dst_path));
+
+			/* create the parent directory if needed and valid */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			maybe_create_directory(parent_path);
+		}
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+			maybe_create_directory(src_path);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
@@ -3057,9 +3122,15 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_wal_log_rec *xlrec =
 		(xl_dbase_create_wal_log_rec *) XLogRecGetData(record);
 		char	   *dbpath;
+		char	   *parent_path;
 
 		dbpath = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
 
+		/* create the parent directory if needed and valid */
+		parent_path = pstrdup(dbpath);
+		get_parent_directory(parent_path);
+		maybe_create_directory(parent_path);
+
 		/* Create the database directory with the version file. */
 		CreateDirAndVersionFile(dbpath, xlrec->db_id, xlrec->tablespace_id,
 								true);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 40514ab550..675f578dfe 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -155,8 +155,6 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -169,32 +167,8 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 9e8ab1420d..9134a73d3d 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -42,6 +42,7 @@
 #include "access/xact.h"
 #include "access/xlog_internal.h"
 #include "access/xlogrecovery.h"
+#include "access/xlogutils.h"
 #include "catalog/namespace.h"
 #include "catalog/objectaccess.h"
 #include "catalog/pg_authid.h"
@@ -139,7 +140,6 @@ extern int	CommitSiblings;
 extern char *default_tablespace;
 extern char *temp_tablespaces;
 extern bool ignore_checksum_failure;
-extern bool ignore_invalid_pages;
 extern bool synchronize_seqscans;
 
 #ifdef TRACE_SYNCSCAN
@@ -1304,10 +1304,12 @@ static struct config_bool ConfigureNamesBool[] =
 		{"ignore_invalid_pages", PGC_POSTMASTER, DEVELOPER_OPTIONS,
 			gettext_noop("Continues recovery after an invalid pages failure."),
 			gettext_noop("Detection of WAL records having references to "
-						 "invalid pages during recovery causes PostgreSQL to "
+						 "invalid pages or WAL records resulting in invalid "
+						 "directory operations during "
+						 "recovery that cause PostgreSQL"
 						 "raise a PANIC-level error, aborting the recovery. "
 						 "Setting ignore_invalid_pages to true causes "
-						 "the system to ignore invalid page references "
+						 "the system to ignore those inconsistencies "
 						 "in WAL records (but still report a warning), "
 						 "and continue recovery. This behavior may cause "
 						 "crashes, data loss, propagate or hide corruption, "
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 64708949db..d88661997f 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -54,6 +54,8 @@ typedef enum
 
 extern HotStandbyState standbyState;
 
+extern bool ignore_invalid_pages;
+
 #define InHotStandby (standbyState >= STANDBY_SNAPSHOT_PENDING)
 
 
diff --git a/src/test/recovery/t/029_replay_tsp_drops.pl b/src/test/recovery/t/029_replay_tsp_drops.pl
new file mode 100644
index 0000000000..b401ab8072
--- /dev/null
+++ b/src/test/recovery/t/029_replay_tsp_drops.pl
@@ -0,0 +1,155 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+#
+# Tests relating to PostgreSQL crash recovery and redo
+#
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_tablespace
+{
+	my ($strategy) = @_;
+
+	my $node_primary = PostgreSQL::Test::Cluster->new("primary1_$strategy");
+	$node_primary->init(allows_streaming => 1);
+	$node_primary->start;
+	$node_primary->psql('postgres',
+			qq[
+				SET allow_in_place_tablespaces=on;
+				CREATE TABLESPACE dropme_ts1 LOCATION '';
+				CREATE TABLESPACE dropme_ts2 LOCATION '';
+				CREATE TABLESPACE source_ts  LOCATION '';
+				CREATE TABLESPACE target_ts  LOCATION '';
+				CREATE DATABASE template_db IS_TEMPLATE = true;
+			]);
+	my $backup_name = 'my_backup';
+	$node_primary->backup($backup_name);
+
+	my $node_standby = PostgreSQL::Test::Cluster->new("standby2_$strategy");
+	$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+	$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");	
+	$node_standby->start;
+
+	# Make sure connection is made
+	$node_primary->poll_query_until(
+		'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+	$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+	# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+	# to be applied to already-removed directories.
+	my $query = q[
+	CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1 STRATEGY=<STRATEGY>;
+	CREATE TABLE t (a int) TABLESPACE dropme_ts2;
+	CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2 STRATEGY=<STRATEGY>;
+	CREATE DATABASE moveme_db TABLESPACE source_ts STRATEGY=<STRATEGY>;
+	ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+	CREATE DATABASE newdb TEMPLATE template_db STRATEGY=<STRATEGY>;
+	ALTER DATABASE template_db IS_TEMPLATE = false;
+	DROP DATABASE dropme_db1;
+	DROP TABLE t;
+	DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+	DROP TABLESPACE source_ts;
+	DROP DATABASE template_db;];
+
+	$query =~ s/<STRATEGY>/$strategy/g;
+	$node_primary->safe_psql('postgres', $query);
+	$node_primary->wait_for_catchup($node_standby, 'replay',
+									$node_primary->lsn('replay'));
+
+	# show "create missing directory" log message
+	$node_standby->safe_psql('postgres',
+							 "ALTER SYSTEM SET log_min_messages TO debug1;");
+	$node_standby->stop('immediate');
+	# Should restart ignoring directory creation error.
+	is($node_standby->start(fail_ok => 1), 1);
+	$node_standby->stop('immediate');
+}	
+
+test_tablespace("FILE_COPY");
+test_tablespace("WAL_LOG");
+
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).  This is
+# effective only for CREATE DATABASE WITH STRATEGY=FILE_COPY.
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+$node_primary->safe_psql('postgres', q[
+						 SET allow_in_place_tablespaces=on;
+						 CREATE TABLESPACE ts1 LOCATION '']);
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 WITH TABLESPACE ts1 STRATEGY=FILE_COPY");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");	
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+my $tspoid = $node_standby->safe_psql('postgres',
+									  "SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
+my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
+File::Path::rmtree($tspdir);
+
+my $logstart = get_log_size($node_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1 STRATEGY=FILE_COPY;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+# In this test, PANIC turns into WARNING by ignore_invalid_pages.
+# Check the log messages instead of confirming standby failure.
+my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
+while ($max_attempts-- >= 0)
+{
+	last if (find_in_log(
+				 $node_standby,
+				 "WARNING:  creating missing directory: pg_tblspc/",
+				 $logstart));
+	sleep 1;
+}
+ok($max_attempts > 0, "invalid directory creation is detected");
+
+done_testing();
+
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
-- 
2.27.0

From de2c77f5e2c31c911674377a51359ba8fe662c96 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Wed, 13 Jul 2022 18:14:18 +0200
Subject: [PATCH v24] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch allows missing tablespaces to be created during recovery
before reaching consistency.  The tablespaces are created as real
directories that should not exists but will be removed until reaching
consistency. CheckRecoveryConsistency is responsible to make sure they
have disappeared.

Similar to log_invalid_page mechanism, the GUC ignore_invalid_pages
turns into PANIC errors detected by this patch into WARNING, which
allows continueing recovery.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 doc/src/sgml/config.sgml                    |   5 +-
 src/backend/access/transam/xlogrecovery.c   |  59 ++++++++
 src/backend/commands/dbcommands.c           |  71 +++++++++
 src/backend/commands/tablespace.c           |  28 +---
 src/backend/utils/misc/guc.c                |   8 +-
 src/include/access/xlogutils.h              |   2 +
 src/test/recovery/t/029_replay_tsp_drops.pl | 155 ++++++++++++++++++++
 7 files changed, 296 insertions(+), 32 deletions(-)
 create mode 100644 src/test/recovery/t/029_replay_tsp_drops.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..1e1c8c1cb7 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -11363,11 +11363,12 @@ LOG:  CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1)
       <listitem>
        <para>
         If set to <literal>off</literal> (the default), detection of
-        WAL records having references to invalid pages during
+        WAL records having references to invalid pages or
+        WAL records resulting in invalid directory operations during
         recovery causes <productname>PostgreSQL</productname> to
         raise a PANIC-level error, aborting the recovery. Setting
         <varname>ignore_invalid_pages</varname> to <literal>on</literal>
-        causes the system to ignore invalid page references in WAL records
+        causes the system to ignore invalid actions caused by such WAL records
         (but still report a warning), and continue the recovery.
         This behavior may <emphasis>cause crashes, data loss,
         propagate or hide corruption, or other serious problems</emphasis>.
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..ae81244e06 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2008,6 +2008,57 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Makes sure that ./pg_tblspc directory doesn't contain a real directory.
+ *
+ * This is intended to be called after reaching consistency.
+ * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
+ * recovery can continue.
+ *
+ * This can't be checked in allow_in_place_tablespaces mode, so skip it in
+ * that case.
+ */
+static void
+CheckTablespaceDirectory(void)
+{
+	char *tblspc_path = "./pg_tblspc";
+	DIR		   *dir;
+	struct dirent *de;
+
+	/* Do not check for this when test tablespaces are in use */
+	if (allow_in_place_tablespaces)
+		return;
+
+	dir = AllocateDir(tblspc_path);
+	while ((de = ReadDir(dir, tblspc_path)) != NULL)
+	{
+		char	path[MAXPGPATH];
+		char   *p;
+#ifndef WIN32
+		struct stat st;
+#endif
+
+		/* Skip entries of non-oid names */
+		for (p = de->d_name; *p && isdigit(*p); p++);
+		if (*p)
+			continue;
+
+		snprintf(path, MAXPGPATH, "%s/%s", tblspc_path, de->d_name);
+
+#ifndef WIN32
+		if (lstat(path, &st) < 0)
+			ereport(ERROR, errcode_for_file_access(),
+					errmsg("could not stat file \"%s\": %m", path));
+
+		if (!S_ISLNK(st.st_mode))
+#else
+		if (!pgwin32_is_junction(path))
+#endif
+			elog(ignore_invalid_pages ? WARNING : PANIC,
+				 "real directory found in pg_tblspc directory: %s", de->d_name);
+	}
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
@@ -2072,6 +2123,14 @@ CheckRecoveryConsistency(void)
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
 						LSN_FORMAT_ARGS(lastReplayedEndRecPtr))));
+
+		/*
+		 * Check that pg_tblspc doesn't contain a real
+		 * directory. Database/CREATE_* records may create a tablespace
+		 * directory that should have been removed until consistency is
+		 * reached.
+		 */
+		CheckTablespaceDirectory();
 	}
 
 	/*
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 1901b434c5..0f860030fa 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -47,6 +48,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -62,6 +64,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/guc.h"
 #include "utils/pg_locale.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -135,6 +138,7 @@ static void CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid,
 									bool isRedo);
 static void CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dboid, Oid src_tsid,
 										Oid dst_tsid);
+static void maybe_create_directory(char *path);
 
 /*
  * Create a new database using the WAL_LOG strategy.
@@ -3008,6 +3012,43 @@ get_database_name(Oid dbid)
 	return result;
 }
 
+/*
+ * maybe_create_directory()
+ *
+ * During recovery, there's a case where we validly need to recover a missing
+ * tablespace directory so that recovery can continue.  This happens when
+ * recovery wants to create a database but the holding tablespace has been
+ * removed before the server stopped.  Since we expect that the directory will
+ * be gone before reaching recovery consistency, and we have no knowledge about
+ * the tablespace other than its OID here, we create a real directory under
+ * pg_tblspc here instead of restoring the symlink.  ignore_invalid_pages=on
+ * reduces the error level so that recovery can continue.
+ */
+static void
+maybe_create_directory(char *path)
+{
+	struct stat	st;
+
+	Assert(RecoveryInProgress());
+
+	if (stat(path, &st) == 0)
+		return;
+
+	/* XXX: Do we make sure that the path is under pg_tblspc? */
+
+	if (reachedConsistency && !ignore_invalid_pages)
+		ereport(PANIC,
+				errmsg("missing directory \"%s\"", path));
+
+	elog(reachedConsistency ? WARNING : DEBUG1,
+		 "creating missing directory: %s", path);
+
+	if (pg_mkdir_p(path, pg_dir_create_mode) != 0)
+		ereport(PANIC,
+				errmsg("could not create missing directory \"%s\": %m", path));
+}
+
+
 /*
  * DATABASE resource manager's routines
  */
@@ -3044,6 +3085,30 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		if (stat(dst_path, &st) < 0)
+		{
+			char *parent_path;
+
+			if (errno != ENOENT)
+				ereport(FATAL,
+						errmsg("could not stat directory \"%s\": %m",
+							   dst_path));
+
+			/* create the parent directory if needed and valid */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			maybe_create_directory(parent_path);
+		}
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+			maybe_create_directory(src_path);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
@@ -3068,9 +3133,15 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_wal_log_rec *xlrec =
 		(xl_dbase_create_wal_log_rec *) XLogRecGetData(record);
 		char	   *dbpath;
+		char	   *parent_path;
 
 		dbpath = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
 
+		/* create the parent directory if needed and valid */
+		parent_path = pstrdup(dbpath);
+		get_parent_directory(parent_path);
+		maybe_create_directory(parent_path);
+
 		/* Create the database directory with the version file. */
 		CreateDirAndVersionFile(dbpath, xlrec->db_id, xlrec->tablespace_id,
 								true);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index c8bdd9992a..ddb031a83f 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -156,8 +156,6 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -170,32 +168,8 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0328029d43..cd6fa84e22 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -43,6 +43,7 @@
 #include "access/xlog_internal.h"
 #include "access/xlogprefetcher.h"
 #include "access/xlogrecovery.h"
+#include "access/xlogutils.h"
 #include "catalog/namespace.h"
 #include "catalog/objectaccess.h"
 #include "catalog/pg_authid.h"
@@ -141,7 +142,6 @@ extern int	CommitSiblings;
 extern char *default_tablespace;
 extern char *temp_tablespaces;
 extern bool ignore_checksum_failure;
-extern bool ignore_invalid_pages;
 extern bool synchronize_seqscans;
 
 #ifdef TRACE_SYNCSCAN
@@ -1336,10 +1336,12 @@ static struct config_bool ConfigureNamesBool[] =
 		{"ignore_invalid_pages", PGC_POSTMASTER, DEVELOPER_OPTIONS,
 			gettext_noop("Continues recovery after an invalid pages failure."),
 			gettext_noop("Detection of WAL records having references to "
-						 "invalid pages during recovery causes PostgreSQL to "
+						 "invalid pages or WAL records resulting in invalid "
+						 "directory operations during "
+						 "recovery that cause PostgreSQL"
 						 "raise a PANIC-level error, aborting the recovery. "
 						 "Setting ignore_invalid_pages to true causes "
-						 "the system to ignore invalid page references "
+						 "the system to ignore those inconsistencies "
 						 "in WAL records (but still report a warning), "
 						 "and continue recovery. This behavior may cause "
 						 "crashes, data loss, propagate or hide corruption, "
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index ef182977bf..f203fdf539 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -54,6 +54,8 @@ typedef enum
 
 extern PGDLLIMPORT HotStandbyState standbyState;
 
+extern bool ignore_invalid_pages;
+
 #define InHotStandby (standbyState >= STANDBY_SNAPSHOT_PENDING)
 
 
diff --git a/src/test/recovery/t/029_replay_tsp_drops.pl b/src/test/recovery/t/029_replay_tsp_drops.pl
new file mode 100644
index 0000000000..d537fe0ce5
--- /dev/null
+++ b/src/test/recovery/t/029_replay_tsp_drops.pl
@@ -0,0 +1,155 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+#
+# Tests relating to PostgreSQL crash recovery and redo
+#
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_tablespace
+{
+	my ($strategy) = @_;
+
+	my $node_primary = PostgreSQL::Test::Cluster->new("primary1_$strategy");
+	$node_primary->init(allows_streaming => 1);
+	$node_primary->start;
+	$node_primary->psql('postgres',
+			qq[
+				SET allow_in_place_tablespaces=on;
+				CREATE TABLESPACE dropme_ts1 LOCATION '';
+				CREATE TABLESPACE dropme_ts2 LOCATION '';
+				CREATE TABLESPACE source_ts  LOCATION '';
+				CREATE TABLESPACE target_ts  LOCATION '';
+				CREATE DATABASE template_db IS_TEMPLATE = true;
+			]);
+	my $backup_name = 'my_backup';
+	$node_primary->backup($backup_name);
+
+	my $node_standby = PostgreSQL::Test::Cluster->new("standby2_$strategy");
+	$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+	$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");
+	$node_standby->start;
+
+	# Make sure connection is made
+	$node_primary->poll_query_until(
+		'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+	$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+	# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+	# to be applied to already-removed directories.
+	my $query = q[
+	CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1 STRATEGY=<STRATEGY>;
+	CREATE TABLE t (a int) TABLESPACE dropme_ts2;
+	CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2 STRATEGY=<STRATEGY>;
+	CREATE DATABASE moveme_db TABLESPACE source_ts STRATEGY=<STRATEGY>;
+	ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+	CREATE DATABASE newdb TEMPLATE template_db STRATEGY=<STRATEGY>;
+	ALTER DATABASE template_db IS_TEMPLATE = false;
+	DROP DATABASE dropme_db1;
+	DROP TABLE t;
+	DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+	DROP TABLESPACE source_ts;
+	DROP DATABASE template_db;];
+
+	$query =~ s/<STRATEGY>/$strategy/g;
+	$node_primary->safe_psql('postgres', $query);
+	$node_primary->wait_for_catchup($node_standby, 'replay',
+									$node_primary->lsn('replay'));
+
+	# show "create missing directory" log message
+	$node_standby->safe_psql('postgres',
+							 "ALTER SYSTEM SET log_min_messages TO debug1;");
+	$node_standby->stop('immediate');
+	# Should restart ignoring directory creation error.
+	is($node_standby->start(fail_ok => 1), 1);
+	$node_standby->stop('immediate');
+}
+
+test_tablespace("FILE_COPY");
+test_tablespace("WAL_LOG");
+
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).  This is
+# effective only for CREATE DATABASE WITH STRATEGY=FILE_COPY.
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+$node_primary->safe_psql('postgres', q[
+						 SET allow_in_place_tablespaces=on;
+						 CREATE TABLESPACE ts1 LOCATION '']);
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 WITH TABLESPACE ts1 STRATEGY=FILE_COPY");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+my $tspoid = $node_standby->safe_psql('postgres',
+									  "SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
+my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
+File::Path::rmtree($tspdir);
+
+my $logstart = get_log_size($node_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1 STRATEGY=FILE_COPY;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+# In this test, PANIC turns into WARNING by ignore_invalid_pages.
+# Check the log messages instead of confirming standby failure.
+my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
+while ($max_attempts-- >= 0)
+{
+	last if (find_in_log(
+				 $node_standby,
+				 "WARNING:  creating missing directory: pg_tblspc/",
+				 $logstart));
+	sleep 1;
+}
+ok($max_attempts > 0, "invalid directory creation is detected");
+
+done_testing();
+
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
-- 
2.30.2

From bdb691c2a86301e5369b3ae7f735fa5f7c29304d Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Wed, 13 Jul 2022 18:14:18 +0200
Subject: [PATCH v25 1/4] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch allows missing tablespaces to be created during recovery
before reaching consistency.  The tablespaces are created as real
directories that should not exists but will be removed until reaching
consistency. CheckRecoveryConsistency is responsible to make sure they
have disappeared.

Similar to log_invalid_page mechanism, the GUC ignore_invalid_pages
turns into PANIC errors detected by this patch into WARNING, which
allows continueing recovery.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 doc/src/sgml/config.sgml                    |   5 +-
 src/backend/access/transam/xlogrecovery.c   |  59 ++++++++
 src/backend/commands/dbcommands.c           |  71 +++++++++
 src/backend/commands/tablespace.c           |  28 +---
 src/backend/utils/misc/guc.c                |   8 +-
 src/include/access/xlogutils.h              |   2 +
 src/test/recovery/t/029_replay_tsp_drops.pl | 155 ++++++++++++++++++++
 7 files changed, 296 insertions(+), 32 deletions(-)
 create mode 100644 src/test/recovery/t/029_replay_tsp_drops.pl

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..1e1c8c1cb7 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -11363,11 +11363,12 @@ LOG:  CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1)
       <listitem>
        <para>
         If set to <literal>off</literal> (the default), detection of
-        WAL records having references to invalid pages during
+        WAL records having references to invalid pages or
+        WAL records resulting in invalid directory operations during
         recovery causes <productname>PostgreSQL</productname> to
         raise a PANIC-level error, aborting the recovery. Setting
         <varname>ignore_invalid_pages</varname> to <literal>on</literal>
-        causes the system to ignore invalid page references in WAL records
+        causes the system to ignore invalid actions caused by such WAL records
         (but still report a warning), and continue the recovery.
         This behavior may <emphasis>cause crashes, data loss,
         propagate or hide corruption, or other serious problems</emphasis>.
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..ae81244e06 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2008,6 +2008,57 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Makes sure that ./pg_tblspc directory doesn't contain a real directory.
+ *
+ * This is intended to be called after reaching consistency.
+ * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
+ * recovery can continue.
+ *
+ * This can't be checked in allow_in_place_tablespaces mode, so skip it in
+ * that case.
+ */
+static void
+CheckTablespaceDirectory(void)
+{
+	char *tblspc_path = "./pg_tblspc";
+	DIR		   *dir;
+	struct dirent *de;
+
+	/* Do not check for this when test tablespaces are in use */
+	if (allow_in_place_tablespaces)
+		return;
+
+	dir = AllocateDir(tblspc_path);
+	while ((de = ReadDir(dir, tblspc_path)) != NULL)
+	{
+		char	path[MAXPGPATH];
+		char   *p;
+#ifndef WIN32
+		struct stat st;
+#endif
+
+		/* Skip entries of non-oid names */
+		for (p = de->d_name; *p && isdigit(*p); p++);
+		if (*p)
+			continue;
+
+		snprintf(path, MAXPGPATH, "%s/%s", tblspc_path, de->d_name);
+
+#ifndef WIN32
+		if (lstat(path, &st) < 0)
+			ereport(ERROR, errcode_for_file_access(),
+					errmsg("could not stat file \"%s\": %m", path));
+
+		if (!S_ISLNK(st.st_mode))
+#else
+		if (!pgwin32_is_junction(path))
+#endif
+			elog(ignore_invalid_pages ? WARNING : PANIC,
+				 "real directory found in pg_tblspc directory: %s", de->d_name);
+	}
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
@@ -2072,6 +2123,14 @@ CheckRecoveryConsistency(void)
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
 						LSN_FORMAT_ARGS(lastReplayedEndRecPtr))));
+
+		/*
+		 * Check that pg_tblspc doesn't contain a real
+		 * directory. Database/CREATE_* records may create a tablespace
+		 * directory that should have been removed until consistency is
+		 * reached.
+		 */
+		CheckTablespaceDirectory();
 	}
 
 	/*
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 1901b434c5..0f860030fa 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -47,6 +48,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -62,6 +64,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/guc.h"
 #include "utils/pg_locale.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -135,6 +138,7 @@ static void CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid,
 									bool isRedo);
 static void CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dboid, Oid src_tsid,
 										Oid dst_tsid);
+static void maybe_create_directory(char *path);
 
 /*
  * Create a new database using the WAL_LOG strategy.
@@ -3008,6 +3012,43 @@ get_database_name(Oid dbid)
 	return result;
 }
 
+/*
+ * maybe_create_directory()
+ *
+ * During recovery, there's a case where we validly need to recover a missing
+ * tablespace directory so that recovery can continue.  This happens when
+ * recovery wants to create a database but the holding tablespace has been
+ * removed before the server stopped.  Since we expect that the directory will
+ * be gone before reaching recovery consistency, and we have no knowledge about
+ * the tablespace other than its OID here, we create a real directory under
+ * pg_tblspc here instead of restoring the symlink.  ignore_invalid_pages=on
+ * reduces the error level so that recovery can continue.
+ */
+static void
+maybe_create_directory(char *path)
+{
+	struct stat	st;
+
+	Assert(RecoveryInProgress());
+
+	if (stat(path, &st) == 0)
+		return;
+
+	/* XXX: Do we make sure that the path is under pg_tblspc? */
+
+	if (reachedConsistency && !ignore_invalid_pages)
+		ereport(PANIC,
+				errmsg("missing directory \"%s\"", path));
+
+	elog(reachedConsistency ? WARNING : DEBUG1,
+		 "creating missing directory: %s", path);
+
+	if (pg_mkdir_p(path, pg_dir_create_mode) != 0)
+		ereport(PANIC,
+				errmsg("could not create missing directory \"%s\": %m", path));
+}
+
+
 /*
  * DATABASE resource manager's routines
  */
@@ -3044,6 +3085,30 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		if (stat(dst_path, &st) < 0)
+		{
+			char *parent_path;
+
+			if (errno != ENOENT)
+				ereport(FATAL,
+						errmsg("could not stat directory \"%s\": %m",
+							   dst_path));
+
+			/* create the parent directory if needed and valid */
+			parent_path = pstrdup(dst_path);
+			get_parent_directory(parent_path);
+			maybe_create_directory(parent_path);
+		}
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+			maybe_create_directory(src_path);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
@@ -3068,9 +3133,15 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_wal_log_rec *xlrec =
 		(xl_dbase_create_wal_log_rec *) XLogRecGetData(record);
 		char	   *dbpath;
+		char	   *parent_path;
 
 		dbpath = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
 
+		/* create the parent directory if needed and valid */
+		parent_path = pstrdup(dbpath);
+		get_parent_directory(parent_path);
+		maybe_create_directory(parent_path);
+
 		/* Create the database directory with the version file. */
 		CreateDirAndVersionFile(dbpath, xlrec->db_id, xlrec->tablespace_id,
 								true);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index c8bdd9992a..ddb031a83f 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -156,8 +156,6 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -170,32 +168,8 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0328029d43..cd6fa84e22 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -43,6 +43,7 @@
 #include "access/xlog_internal.h"
 #include "access/xlogprefetcher.h"
 #include "access/xlogrecovery.h"
+#include "access/xlogutils.h"
 #include "catalog/namespace.h"
 #include "catalog/objectaccess.h"
 #include "catalog/pg_authid.h"
@@ -141,7 +142,6 @@ extern int	CommitSiblings;
 extern char *default_tablespace;
 extern char *temp_tablespaces;
 extern bool ignore_checksum_failure;
-extern bool ignore_invalid_pages;
 extern bool synchronize_seqscans;
 
 #ifdef TRACE_SYNCSCAN
@@ -1336,10 +1336,12 @@ static struct config_bool ConfigureNamesBool[] =
 		{"ignore_invalid_pages", PGC_POSTMASTER, DEVELOPER_OPTIONS,
 			gettext_noop("Continues recovery after an invalid pages failure."),
 			gettext_noop("Detection of WAL records having references to "
-						 "invalid pages during recovery causes PostgreSQL to "
+						 "invalid pages or WAL records resulting in invalid "
+						 "directory operations during "
+						 "recovery that cause PostgreSQL"
 						 "raise a PANIC-level error, aborting the recovery. "
 						 "Setting ignore_invalid_pages to true causes "
-						 "the system to ignore invalid page references "
+						 "the system to ignore those inconsistencies "
 						 "in WAL records (but still report a warning), "
 						 "and continue recovery. This behavior may cause "
 						 "crashes, data loss, propagate or hide corruption, "
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index ef182977bf..f203fdf539 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -54,6 +54,8 @@ typedef enum
 
 extern PGDLLIMPORT HotStandbyState standbyState;
 
+extern bool ignore_invalid_pages;
+
 #define InHotStandby (standbyState >= STANDBY_SNAPSHOT_PENDING)
 
 
diff --git a/src/test/recovery/t/029_replay_tsp_drops.pl b/src/test/recovery/t/029_replay_tsp_drops.pl
new file mode 100644
index 0000000000..d537fe0ce5
--- /dev/null
+++ b/src/test/recovery/t/029_replay_tsp_drops.pl
@@ -0,0 +1,155 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+#
+# Tests relating to PostgreSQL crash recovery and redo
+#
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_tablespace
+{
+	my ($strategy) = @_;
+
+	my $node_primary = PostgreSQL::Test::Cluster->new("primary1_$strategy");
+	$node_primary->init(allows_streaming => 1);
+	$node_primary->start;
+	$node_primary->psql('postgres',
+			qq[
+				SET allow_in_place_tablespaces=on;
+				CREATE TABLESPACE dropme_ts1 LOCATION '';
+				CREATE TABLESPACE dropme_ts2 LOCATION '';
+				CREATE TABLESPACE source_ts  LOCATION '';
+				CREATE TABLESPACE target_ts  LOCATION '';
+				CREATE DATABASE template_db IS_TEMPLATE = true;
+			]);
+	my $backup_name = 'my_backup';
+	$node_primary->backup($backup_name);
+
+	my $node_standby = PostgreSQL::Test::Cluster->new("standby2_$strategy");
+	$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+	$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");
+	$node_standby->start;
+
+	# Make sure connection is made
+	$node_primary->poll_query_until(
+		'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+	$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+	# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+	# to be applied to already-removed directories.
+	my $query = q[
+	CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1 STRATEGY=<STRATEGY>;
+	CREATE TABLE t (a int) TABLESPACE dropme_ts2;
+	CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2 STRATEGY=<STRATEGY>;
+	CREATE DATABASE moveme_db TABLESPACE source_ts STRATEGY=<STRATEGY>;
+	ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+	CREATE DATABASE newdb TEMPLATE template_db STRATEGY=<STRATEGY>;
+	ALTER DATABASE template_db IS_TEMPLATE = false;
+	DROP DATABASE dropme_db1;
+	DROP TABLE t;
+	DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+	DROP TABLESPACE source_ts;
+	DROP DATABASE template_db;];
+
+	$query =~ s/<STRATEGY>/$strategy/g;
+	$node_primary->safe_psql('postgres', $query);
+	$node_primary->wait_for_catchup($node_standby, 'replay',
+									$node_primary->lsn('replay'));
+
+	# show "create missing directory" log message
+	$node_standby->safe_psql('postgres',
+							 "ALTER SYSTEM SET log_min_messages TO debug1;");
+	$node_standby->stop('immediate');
+	# Should restart ignoring directory creation error.
+	is($node_standby->start(fail_ok => 1), 1);
+	$node_standby->stop('immediate');
+}
+
+test_tablespace("FILE_COPY");
+test_tablespace("WAL_LOG");
+
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).  This is
+# effective only for CREATE DATABASE WITH STRATEGY=FILE_COPY.
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+$node_primary->safe_psql('postgres', q[
+						 SET allow_in_place_tablespaces=on;
+						 CREATE TABLESPACE ts1 LOCATION '']);
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 WITH TABLESPACE ts1 STRATEGY=FILE_COPY");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->append_conf('postgresql.conf', "ignore_invalid_pages = on");
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+my $tspoid = $node_standby->safe_psql('postgres',
+									  "SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
+my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
+File::Path::rmtree($tspdir);
+
+my $logstart = get_log_size($node_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1 STRATEGY=FILE_COPY;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+# In this test, PANIC turns into WARNING by ignore_invalid_pages.
+# Check the log messages instead of confirming standby failure.
+my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
+while ($max_attempts-- >= 0)
+{
+	last if (find_in_log(
+				 $node_standby,
+				 "WARNING:  creating missing directory: pg_tblspc/",
+				 $logstart));
+	sleep 1;
+}
+ok($max_attempts > 0, "invalid directory creation is detected");
+
+done_testing();
+
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
-- 
2.30.2

From 87d05718d69cf26d0f2017dc30e30f0f62dcbbff Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 14 Jul 2022 16:43:03 +0200
Subject: [PATCH v25 2/4] split is_path_tslink as a new routine

---
 src/backend/access/transam/xlogrecovery.c | 68 +++++++++++++----------
 1 file changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index ae81244e06..e04d30cf3e 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2008,54 +2008,66 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Is the given directory entry a symlink/junction point?  Subroutine for
+ * CheckTablespaceDirectory.
+ */
+static bool
+is_path_tslink(const char *path)
+{
+#ifndef WIN32
+		struct stat st;
+
+		if (lstat(path, &st) < 0)
+			ereport(ERROR, errcode_for_file_access(),
+					errmsg("could not stat file \"%s\": %m", path));
+		return S_ISLNK(st.st_mode);
+#else
+		return pgwin32_is_junction(path);
+#endif
+}
+
 /*
  * Makes sure that ./pg_tblspc directory doesn't contain a real directory.
  *
- * This is intended to be called after reaching consistency.
- * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
- * recovery can continue.
+ * Replay of database creation XLOG records for databases that were later
+ * dropped can create fake directories in pg_tblspc.  By the time consistency
+ * is reached these directories should have been removed; here we verify
+ * that this did indeed happen.  This must be called after reached consistent
+ * state.
  *
- * This can't be checked in allow_in_place_tablespaces mode, so skip it in
- * that case.
+ * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
+ * recovery can continue.  XXX piggybacking on this particular GUC sounds like
+ * a bad idea.  Why not just advise to use allow_in_place_tablespaces?
  */
 static void
 CheckTablespaceDirectory(void)
 {
-	char *tblspc_path = "./pg_tblspc";
 	DIR		   *dir;
 	struct dirent *de;
 
-	/* Do not check for this when test tablespaces are in use */
+	/*
+	 * In allow_in_place_tablespaces mode, it is valid to have non-symlink
+	 * directories in pg_tblspc, so we cannot run this check.  Give up.
+	 */
 	if (allow_in_place_tablespaces)
 		return;
 
-	dir = AllocateDir(tblspc_path);
-	while ((de = ReadDir(dir, tblspc_path)) != NULL)
+	dir = AllocateDir("pg_tblspc");
+	while ((de = ReadDir(dir, "pg_tblspc")) != NULL)
 	{
-		char	path[MAXPGPATH];
-		char   *p;
-#ifndef WIN32
-		struct stat st;
-#endif
+		char	path[MAXPGPATH + 10];
 
 		/* Skip entries of non-oid names */
-		for (p = de->d_name; *p && isdigit(*p); p++);
-		if (*p)
+		if (strspn(de->d_name, "0123456789") != strlen(de->d_name))
 			continue;
 
-		snprintf(path, MAXPGPATH, "%s/%s", tblspc_path, de->d_name);
+		snprintf(path, sizeof(path), "pg_tblspc/%s", de->d_name);
 
-#ifndef WIN32
-		if (lstat(path, &st) < 0)
-			ereport(ERROR, errcode_for_file_access(),
-					errmsg("could not stat file \"%s\": %m", path));
-
-		if (!S_ISLNK(st.st_mode))
-#else
-		if (!pgwin32_is_junction(path))
-#endif
-			elog(ignore_invalid_pages ? WARNING : PANIC,
-				 "real directory found in pg_tblspc directory: %s", de->d_name);
+		if (!is_path_tslink(path))
+			ereport(ignore_invalid_pages ? WARNING : PANIC,
+					(errcode(ERRCODE_DATA_CORRUPTED),
+					 errmsg("real directory found in pg_tblspc directory: %s", de->d_name)));
 	}
 }
 
-- 
2.30.2

From 0e03c82ee07569b869b382fac83d98d0b5d5d870 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 14 Jul 2022 23:39:08 +0200
Subject: [PATCH v25 3/4] Don't rely on ignore_invalid_pages at all

Since a workaround with allow_in_place_tablespaces is possible,
there doesn't seem to be a need for the ignore_invalid_pages one.
Remove it.
---
 src/backend/access/transam/xlogrecovery.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index e04d30cf3e..b0ae63fbac 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2035,10 +2035,6 @@ is_path_tslink(const char *path)
  * is reached these directories should have been removed; here we verify
  * that this did indeed happen.  This must be called after reached consistent
  * state.
- *
- * ignore_invalid_pages=on turns into the PANIC error into WARNING so that
- * recovery can continue.  XXX piggybacking on this particular GUC sounds like
- * a bad idea.  Why not just advise to use allow_in_place_tablespaces?
  */
 static void
 CheckTablespaceDirectory(void)
@@ -2065,7 +2061,7 @@ CheckTablespaceDirectory(void)
 		snprintf(path, sizeof(path), "pg_tblspc/%s", de->d_name);
 
 		if (!is_path_tslink(path))
-			ereport(ignore_invalid_pages ? WARNING : PANIC,
+			ereport(PANIC,
 					(errcode(ERRCODE_DATA_CORRUPTED),
 					 errmsg("real directory found in pg_tblspc directory: %s", de->d_name)));
 	}
-- 
2.30.2

From 427186297c41e38c61ebb73d07e962400c7ccc22 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 14 Jul 2022 16:43:18 +0200
Subject: [PATCH v25 4/4] do CheckTablespaceDirectory first

---
 src/backend/access/transam/xlogrecovery.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index b0ae63fbac..c457c36daa 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -2127,18 +2127,18 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check that pg_tblspc doesn't contain any real directories.
+		 * Replay of Database/CREATE_* records may have created ficticious
+		 * tablespace directories that should have been removed by the time
+		 * consistency was reached.
+		 */
+		CheckTablespaceDirectory();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
 						LSN_FORMAT_ARGS(lastReplayedEndRecPtr))));
-
-		/*
-		 * Check that pg_tblspc doesn't contain a real
-		 * directory. Database/CREATE_* records may create a tablespace
-		 * directory that should have been removed until consistency is
-		 * reached.
-		 */
-		CheckTablespaceDirectory();
 	}
 
 	/*
-- 
2.30.2

From 26a0be53592a20aa09501e9015f77a4b3c3c3302 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Wed, 13 Jul 2022 18:14:18 +0200
Subject: [PATCH v26] Fix replay of create database records on standby

Crash recovery on standby may encounter missing directories when
replaying create database WAL records.  Prior to this patch, the
standby would fail to recover in such a case.  However, the
directories could be legitimately missing.  Consider a sequence of WAL
records as follows:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the tablespace
directory, the standby crashes and has to replay the create database
record again, the crash recovery must be able to move on.

This patch allows missing tablespaces to be created during recovery
before reaching consistency.  The tablespaces are created as real
directories that should not exists but will be removed until reaching
consistency. CheckRecoveryConsistency is responsible to make sure they
have disappeared.

The problems detected by this new code are reported as PANIC, except
when allow_in_place_tablespaces is set to ON, in which case they are
WARNING.  Apart from making tests possible, this gives users an escape
hatch in case things don't go as planned.

Diagnosed-by: Paul Guo <paulguo@gmail.com>
Author: Paul Guo <paulguo@gmail.com>
Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 src/backend/access/transam/xlogrecovery.c   |  54 +++++++
 src/backend/commands/dbcommands.c           |  77 ++++++++++
 src/backend/commands/tablespace.c           |  28 +---
 src/test/recovery/t/033_replay_tsp_drops.pl | 155 ++++++++++++++++++++
 4 files changed, 287 insertions(+), 27 deletions(-)
 create mode 100644 src/test/recovery/t/033_replay_tsp_drops.pl

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..850ab6d7e6 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -42,6 +42,7 @@
 #include "access/xlogutils.h"
 #include "catalog/pg_control.h"
 #include "commands/tablespace.h"
+#include "common/file_utils.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/bgwriter.h"
@@ -2008,6 +2009,51 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Verify that, in non-test mode, ./pg_tblspc doesn't contain any real
+ * directories.
+ *
+ * Replay of database creation XLOG records for databases that were later
+ * dropped can create fake directories in pg_tblspc.  By the time consistency
+ * is reached these directories should have been removed; here we verify
+ * that this did indeed happen.  This is to be called at the point where
+ * consistent state is reached.
+ *
+ * allow_in_place_tablespaces turns the PANIC into a WARNING, which is
+ * useful for testing purposes, and also allows for an escape hatch in case
+ * things go south.
+ */
+static void
+CheckTablespaceDirectory(void)
+{
+	DIR		   *dir;
+	struct dirent *de;
+
+	dir = AllocateDir("pg_tblspc");
+	while ((de = ReadDir(dir, "pg_tblspc")) != NULL)
+	{
+		char		path[MAXPGPATH + 10];
+
+		/* Skip entries of non-oid names */
+		if (strspn(de->d_name, "0123456789") != strlen(de->d_name))
+			continue;
+
+		snprintf(path, sizeof(path), "pg_tblspc/%s", de->d_name);
+
+#ifdef WIN32
+		if (!pgwin32_is_junction(path))
+#else
+		if (get_dirent_type(path, de, false, ERROR) != PGFILETYPE_LNK)
+#endif
+			ereport(allow_in_place_tablespaces ? WARNING : PANIC,
+					(errcode(ERRCODE_DATA_CORRUPTED),
+					 errmsg("unexpected directory entry \"%s\" found in %s",
+							de->d_name, "pg_tblspc/"),
+					 errdetail("All directory entries in pg_tblspc/ should be symbolic links."),
+					 errhint("Remove those directories, or set allow_in_place_tablespaces to ON transiently to let recovery complete.")));
+	}
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
@@ -2068,6 +2114,14 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check that pg_tblspc doesn't contain any real directories. Replay
+		 * of Database/CREATE_* records may have created ficticious tablespace
+		 * directories that should have been removed by the time consistency
+		 * was reached.
+		 */
+		CheckTablespaceDirectory();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 099d369b2f..95844bbb69 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -47,6 +48,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -62,6 +64,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/guc.h"
 #include "utils/pg_locale.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -135,6 +138,7 @@ static void CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid,
 									bool isRedo);
 static void CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dboid, Oid src_tsid,
 										Oid dst_tsid);
+static void recovery_create_dbdir(char *path, bool only_tblspc);
 
 /*
  * Create a new database using the WAL_LOG strategy.
@@ -2995,6 +2999,45 @@ get_database_name(Oid dbid)
 	return result;
 }
 
+/*
+ * recovery_create_dbdir()
+ *
+ * During recovery, there's a case where we validly need to recover a missing
+ * tablespace directory so that recovery can continue.  This happens when
+ * recovery wants to create a database but the holding tablespace has been
+ * removed before the server stopped.  Since we expect that the directory will
+ * be gone before reaching recovery consistency, and we have no knowledge about
+ * the tablespace other than its OID here, we create a real directory under
+ * pg_tblspc here instead of restoring the symlink.
+ *
+ * If only_tblspc is true, then the requested directory must be in pg_tblspc/
+ */
+static void
+recovery_create_dbdir(char *path, bool only_tblspc)
+{
+	struct stat st;
+
+	Assert(RecoveryInProgress());
+
+	if (stat(path, &st) == 0)
+		return;
+
+	if (only_tblspc && strstr(path, "pg_tblspc/") == NULL)
+		elog(PANIC, "requested to created invalid directory: %s", path);
+
+	if (reachedConsistency && !allow_in_place_tablespaces)
+		ereport(PANIC,
+				errmsg("missing directory \"%s\"", path));
+
+	elog(reachedConsistency ? WARNING : DEBUG1,
+		 "creating missing directory: %s", path);
+
+	if (pg_mkdir_p(path, pg_dir_create_mode) != 0)
+		ereport(PANIC,
+				errmsg("could not create missing directory \"%s\": %m", path));
+}
+
+
 /*
  * DATABASE resource manager's routines
  */
@@ -3012,6 +3055,7 @@ dbase_redo(XLogReaderState *record)
 		(xl_dbase_create_file_copy_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
@@ -3031,6 +3075,33 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		/*
+		 * If the parent of the target path doesn't exist, create it now. This
+		 * enables us to create the target underneath later.
+		 */
+		parent_path = pstrdup(dst_path);
+		get_parent_directory(parent_path);
+		if (stat(parent_path, &st) < 0)
+		{
+			if (errno != ENOENT)
+				ereport(FATAL,
+						errmsg("could not stat directory \"%s\": %m",
+							   dst_path));
+
+			/* create the parent directory if needed and valid */
+			recovery_create_dbdir(parent_path, true);
+		}
+		pfree(parent_path);
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+			recovery_create_dbdir(src_path, false);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
@@ -3055,9 +3126,15 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_wal_log_rec *xlrec =
 		(xl_dbase_create_wal_log_rec *) XLogRecGetData(record);
 		char	   *dbpath;
+		char	   *parent_path;
 
 		dbpath = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
 
+		/* create the parent directory if needed and valid */
+		parent_path = pstrdup(dbpath);
+		get_parent_directory(parent_path);
+		recovery_create_dbdir(parent_path, true);
+
 		/* Create the database directory with the version file. */
 		CreateDirAndVersionFile(dbpath, xlrec->db_id, xlrec->tablespace_id,
 								true);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index cb7d46089a..a23097399e 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -156,8 +156,6 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -170,32 +168,8 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 					 * continue by creating simple parent directories rather
 					 * than a symlink.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
diff --git a/src/test/recovery/t/033_replay_tsp_drops.pl b/src/test/recovery/t/033_replay_tsp_drops.pl
new file mode 100644
index 0000000000..0986df45e6
--- /dev/null
+++ b/src/test/recovery/t/033_replay_tsp_drops.pl
@@ -0,0 +1,155 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test replay of tablespace/database creation/drop
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_tablespace
+{
+	my ($strategy) = @_;
+
+	my $node_primary = PostgreSQL::Test::Cluster->new("primary1_$strategy");
+	$node_primary->init(allows_streaming => 1);
+	$node_primary->start;
+	$node_primary->psql('postgres',
+			qq[
+				SET allow_in_place_tablespaces=on;
+				CREATE TABLESPACE dropme_ts1 LOCATION '';
+				CREATE TABLESPACE dropme_ts2 LOCATION '';
+				CREATE TABLESPACE source_ts  LOCATION '';
+				CREATE TABLESPACE target_ts  LOCATION '';
+				CREATE DATABASE template_db IS_TEMPLATE = true;
+			]);
+	my $backup_name = 'my_backup';
+	$node_primary->backup($backup_name);
+
+	my $node_standby = PostgreSQL::Test::Cluster->new("standby2_$strategy");
+	$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+	$node_standby->append_conf('postgresql.conf', "allow_in_place_tablespaces = on");
+	$node_standby->start;
+
+	# Make sure connection is made
+	$node_primary->poll_query_until(
+		'postgres', 'SELECT count(*) = 1 FROM pg_stat_replication');
+
+	$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+	# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+	# to be applied to already-removed directories.
+	my $query = q[
+	CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1 STRATEGY=<STRATEGY>;
+	CREATE TABLE t (a int) TABLESPACE dropme_ts2;
+	CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2 STRATEGY=<STRATEGY>;
+	CREATE DATABASE moveme_db TABLESPACE source_ts STRATEGY=<STRATEGY>;
+	ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+	CREATE DATABASE newdb TEMPLATE template_db STRATEGY=<STRATEGY>;
+	ALTER DATABASE template_db IS_TEMPLATE = false;
+	DROP DATABASE dropme_db1;
+	DROP TABLE t;
+	DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+	DROP TABLESPACE source_ts;
+	DROP DATABASE template_db;];
+
+	$query =~ s/<STRATEGY>/$strategy/g;
+	$node_primary->safe_psql('postgres', $query);
+	$node_primary->wait_for_catchup($node_standby, 'replay',
+									$node_primary->lsn('replay'));
+
+	# show "create missing directory" log message
+	$node_standby->safe_psql('postgres',
+							 "ALTER SYSTEM SET log_min_messages TO debug1;");
+	$node_standby->stop('immediate');
+	# Should restart ignoring directory creation error.
+	is($node_standby->start(fail_ok => 1), 1, "standby node started for $strategy");
+	$node_standby->stop('immediate');
+}
+
+test_tablespace("FILE_COPY");
+test_tablespace("WAL_LOG");
+
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).  This is
+# effective only for CREATE DATABASE WITH STRATEGY=FILE_COPY.
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+$node_primary->safe_psql('postgres', q[
+						 SET allow_in_place_tablespaces=on;
+						 CREATE TABLESPACE ts1 LOCATION '']);
+$node_primary->safe_psql('postgres', "CREATE DATABASE db1 WITH TABLESPACE ts1 STRATEGY=FILE_COPY");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1);
+$node_standby->append_conf('postgresql.conf', "allow_in_place_tablespaces = on");
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+my $tspoid = $node_standby->safe_psql('postgres',
+									  "SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
+my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
+File::Path::rmtree($tspdir);
+
+my $logstart = get_log_size($node_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql('postgres',
+						q[CREATE TABLE should_not_replay_insertion(a int);
+						  CREATE DATABASE db2 WITH TABLESPACE ts1 STRATEGY=FILE_COPY;
+						  INSERT INTO should_not_replay_insertion VALUES (1);]);
+
+# Standby should fail and should not silently skip replaying the wal
+# In this test, PANIC turns into WARNING by allow_in_place_tablespaces.
+# Check the log messages instead of confirming standby failure.
+my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
+while ($max_attempts-- >= 0)
+{
+	last if (find_in_log(
+				 $node_standby,
+				 "WARNING:  creating missing directory: pg_tblspc/",
+				 $logstart));
+	sleep 1;
+}
+ok($max_attempts > 0, "invalid directory creation is detected");
+
+done_testing();
+
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
-- 
2.30.2

From b84f66975d664c45babd878a43c67601b7f7d2b6 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Wed, 27 Jul 2022 20:22:21 +0200
Subject: [PATCH v27] Fix replay of create database records on standby
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Crash recovery on standby may encounter missing directories
when replaying database-creation WAL records.  Prior to this
patch, the standby would fail to recover in such a case;
however, the directories could be legitimately missing.
Consider the following sequence of commands:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the
tablespace directory, the standby crashes and has to replay the
create database record again, crash recovery must be able to continue.

A fix for this problem was already attempted in 49d9cfc68bf4, but it
was reverted because of design issues.  This new version is based
on Robert Haas' proposal: any missing tablespaces are created
during recovery before reaching consistency.  Tablespaces
are created as real directories, and should be deleted
by later replay.  CheckRecoveryConsistency ensures
they have disappeared.

The problems detected by this new code are reported as PANIC,
except when allow_in_place_tablespaces is set to ON, in which
case they are WARNING.  Apart from making tests possible, this
gives users an escape hatch in case things don't go as planned.

Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Author: Paul Guo <paulguo@gmail.com>
Reviewed-by: Anastasia Lubennikova <lubennikovaav@gmail.com> (older versions)
Reviewed-by: Fujii Masao <masao.fujii@oss.nttdata.com> (older versions)
Reviewed-by: Michaël Paquier <michael@paquier.xyz>
Diagnosed-by: Paul Guo <paulguo@gmail.com>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
---
 src/backend/access/transam/xlogrecovery.c   |  50 ++++++
 src/backend/commands/dbcommands.c           |  77 +++++++++
 src/backend/commands/tablespace.c           |  40 ++---
 src/test/recovery/t/033_replay_tsp_drops.pl | 169 ++++++++++++++++++++
 4 files changed, 305 insertions(+), 31 deletions(-)
 create mode 100644 src/test/recovery/t/033_replay_tsp_drops.pl

diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index e383c2123a..27e02fbfcd 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -42,6 +42,7 @@
 #include "access/xlogutils.h"
 #include "catalog/pg_control.h"
 #include "commands/tablespace.h"
+#include "common/file_utils.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/bgwriter.h"
@@ -2008,6 +2009,47 @@ xlogrecovery_redo(XLogReaderState *record, TimeLineID replayTLI)
 	}
 }
 
+/*
+ * Verify that, in non-test mode, ./pg_tblspc doesn't contain any real
+ * directories.
+ *
+ * Replay of database creation XLOG records for databases that were later
+ * dropped can create fake directories in pg_tblspc.  By the time consistency
+ * is reached these directories should have been removed; here we verify
+ * that this did indeed happen.  This is to be called at the point where
+ * consistent state is reached.
+ *
+ * allow_in_place_tablespaces turns the PANIC into a WARNING, which is
+ * useful for testing purposes, and also allows for an escape hatch in case
+ * things go south.
+ */
+static void
+CheckTablespaceDirectory(void)
+{
+	DIR		   *dir;
+	struct dirent *de;
+
+	dir = AllocateDir("pg_tblspc");
+	while ((de = ReadDir(dir, "pg_tblspc")) != NULL)
+	{
+		char		path[MAXPGPATH + 10];
+
+		/* Skip entries of non-oid names */
+		if (strspn(de->d_name, "0123456789") != strlen(de->d_name))
+			continue;
+
+		snprintf(path, sizeof(path), "pg_tblspc/%s", de->d_name);
+
+		if (get_dirent_type(path, de, false, ERROR) != PGFILETYPE_LNK)
+			ereport(allow_in_place_tablespaces ? WARNING : PANIC,
+					(errcode(ERRCODE_DATA_CORRUPTED),
+					 errmsg("unexpected directory entry \"%s\" found in %s",
+							de->d_name, "pg_tblspc/"),
+					 errdetail("All directory entries in pg_tblspc/ should be symbolic links."),
+					 errhint("Remove those directories, or set allow_in_place_tablespaces to ON transiently to let recovery complete.")));
+	}
+}
+
 /*
  * Checks if recovery has reached a consistent state. When consistency is
  * reached and we have a valid starting standby snapshot, tell postmaster
@@ -2068,6 +2110,14 @@ CheckRecoveryConsistency(void)
 		 */
 		XLogCheckInvalidPages();
 
+		/*
+		 * Check that pg_tblspc doesn't contain any real directories. Replay
+		 * of Database/CREATE_* records may have created ficticious tablespace
+		 * directories that should have been removed by the time consistency
+		 * was reached.
+		 */
+		CheckTablespaceDirectory();
+
 		reachedConsistency = true;
 		ereport(LOG,
 				(errmsg("consistent recovery state reached at %X/%X",
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 099d369b2f..95844bbb69 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -30,6 +30,7 @@
 #include "access/tableam.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
+#include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
@@ -47,6 +48,7 @@
 #include "commands/defrem.h"
 #include "commands/seclabel.h"
 #include "commands/tablespace.h"
+#include "common/file_perm.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -62,6 +64,7 @@
 #include "utils/acl.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/guc.h"
 #include "utils/pg_locale.h"
 #include "utils/relmapper.h"
 #include "utils/snapmgr.h"
@@ -135,6 +138,7 @@ static void CreateDirAndVersionFile(char *dbpath, Oid dbid, Oid tsid,
 									bool isRedo);
 static void CreateDatabaseUsingFileCopy(Oid src_dboid, Oid dboid, Oid src_tsid,
 										Oid dst_tsid);
+static void recovery_create_dbdir(char *path, bool only_tblspc);
 
 /*
  * Create a new database using the WAL_LOG strategy.
@@ -2995,6 +2999,45 @@ get_database_name(Oid dbid)
 	return result;
 }
 
+/*
+ * recovery_create_dbdir()
+ *
+ * During recovery, there's a case where we validly need to recover a missing
+ * tablespace directory so that recovery can continue.  This happens when
+ * recovery wants to create a database but the holding tablespace has been
+ * removed before the server stopped.  Since we expect that the directory will
+ * be gone before reaching recovery consistency, and we have no knowledge about
+ * the tablespace other than its OID here, we create a real directory under
+ * pg_tblspc here instead of restoring the symlink.
+ *
+ * If only_tblspc is true, then the requested directory must be in pg_tblspc/
+ */
+static void
+recovery_create_dbdir(char *path, bool only_tblspc)
+{
+	struct stat st;
+
+	Assert(RecoveryInProgress());
+
+	if (stat(path, &st) == 0)
+		return;
+
+	if (only_tblspc && strstr(path, "pg_tblspc/") == NULL)
+		elog(PANIC, "requested to created invalid directory: %s", path);
+
+	if (reachedConsistency && !allow_in_place_tablespaces)
+		ereport(PANIC,
+				errmsg("missing directory \"%s\"", path));
+
+	elog(reachedConsistency ? WARNING : DEBUG1,
+		 "creating missing directory: %s", path);
+
+	if (pg_mkdir_p(path, pg_dir_create_mode) != 0)
+		ereport(PANIC,
+				errmsg("could not create missing directory \"%s\": %m", path));
+}
+
+
 /*
  * DATABASE resource manager's routines
  */
@@ -3012,6 +3055,7 @@ dbase_redo(XLogReaderState *record)
 		(xl_dbase_create_file_copy_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parent_path;
 		struct stat st;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
@@ -3031,6 +3075,33 @@ dbase_redo(XLogReaderState *record)
 								dst_path)));
 		}
 
+		/*
+		 * If the parent of the target path doesn't exist, create it now. This
+		 * enables us to create the target underneath later.
+		 */
+		parent_path = pstrdup(dst_path);
+		get_parent_directory(parent_path);
+		if (stat(parent_path, &st) < 0)
+		{
+			if (errno != ENOENT)
+				ereport(FATAL,
+						errmsg("could not stat directory \"%s\": %m",
+							   dst_path));
+
+			/* create the parent directory if needed and valid */
+			recovery_create_dbdir(parent_path, true);
+		}
+		pfree(parent_path);
+
+		/*
+		 * There's a case where the copy source directory is missing for the
+		 * same reason above.  Create the emtpy source directory so that
+		 * copydir below doesn't fail.  The directory will be dropped soon by
+		 * recovery.
+		 */
+		if (stat(src_path, &st) < 0 && errno == ENOENT)
+			recovery_create_dbdir(src_path, false);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.
@@ -3055,9 +3126,15 @@ dbase_redo(XLogReaderState *record)
 		xl_dbase_create_wal_log_rec *xlrec =
 		(xl_dbase_create_wal_log_rec *) XLogRecGetData(record);
 		char	   *dbpath;
+		char	   *parent_path;
 
 		dbpath = GetDatabasePath(xlrec->db_id, xlrec->tablespace_id);
 
+		/* create the parent directory if needed and valid */
+		parent_path = pstrdup(dbpath);
+		get_parent_directory(parent_path);
+		recovery_create_dbdir(parent_path, true);
+
 		/* Create the database directory with the version file. */
 		CreateDirAndVersionFile(dbpath, xlrec->db_id, xlrec->tablespace_id,
 								true);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index cb7d46089a..570ce3dbd5 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -156,8 +156,6 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 				/* Directory creation failed? */
 				if (MakePGDirectory(dir) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -166,36 +164,16 @@ TablespaceCreateDbspace(Oid spcOid, Oid dbOid, bool isRedo)
 										dir)));
 
 					/*
-					 * Parent directories are missing during WAL replay, so
-					 * continue by creating simple parent directories rather
-					 * than a symlink.
+					 * During WAL replay, it's conceivable that several levels
+					 * of directories are missing if tablespaces are dropped
+					 * further ahead of the WAL stream than we're currently
+					 * replaying.  An easy way forward is to create them as
+					 * plain directories and hope they are removed by further
+					 * WAL replay if necessary.  If this also fails, there is
+					 * trouble we cannot get out of, so just report that and
+					 * bail out.
 					 */
-
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (MakePGDirectory(parentdir) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-								 errmsg("could not create directory \"%s\": %m",
-										parentdir)));
-					pfree(parentdir);
-
-					/* Create database directory */
-					if (MakePGDirectory(dir) < 0)
+					if (pg_mkdir_p(dir, pg_dir_create_mode) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 								 errmsg("could not create directory \"%s\": %m",
diff --git a/src/test/recovery/t/033_replay_tsp_drops.pl b/src/test/recovery/t/033_replay_tsp_drops.pl
new file mode 100644
index 0000000000..9b74cb09ac
--- /dev/null
+++ b/src/test/recovery/t/033_replay_tsp_drops.pl
@@ -0,0 +1,169 @@
+
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+# Test replay of tablespace/database creation/drop
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_tablespace
+{
+	my ($strategy) = @_;
+
+	my $node_primary = PostgreSQL::Test::Cluster->new("primary1_$strategy");
+	$node_primary->init(allows_streaming => 1);
+	$node_primary->start;
+	$node_primary->psql(
+		'postgres',
+		qq[
+			SET allow_in_place_tablespaces=on;
+			CREATE TABLESPACE dropme_ts1 LOCATION '';
+			CREATE TABLESPACE dropme_ts2 LOCATION '';
+			CREATE TABLESPACE source_ts  LOCATION '';
+			CREATE TABLESPACE target_ts  LOCATION '';
+			CREATE DATABASE template_db IS_TEMPLATE = true;
+		]);
+	my $backup_name = 'my_backup';
+	$node_primary->backup($backup_name);
+
+	my $node_standby = PostgreSQL::Test::Cluster->new("standby2_$strategy");
+	$node_standby->init_from_backup($node_primary, $backup_name,
+		has_streaming => 1);
+	$node_standby->append_conf('postgresql.conf',
+		"allow_in_place_tablespaces = on");
+	$node_standby->start;
+
+	# Make sure connection is made
+	$node_primary->poll_query_until('postgres',
+		'SELECT count(*) = 1 FROM pg_stat_replication');
+
+	$node_standby->safe_psql('postgres', 'CHECKPOINT');
+
+	# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
+	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
+	# to be applied to already-removed directories.
+	my $query = q[
+		CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1 STRATEGY=<STRATEGY>;
+		CREATE TABLE t (a int) TABLESPACE dropme_ts2;
+		CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2 STRATEGY=<STRATEGY>;
+		CREATE DATABASE moveme_db TABLESPACE source_ts STRATEGY=<STRATEGY>;
+		ALTER DATABASE moveme_db SET TABLESPACE target_ts;
+		CREATE DATABASE newdb TEMPLATE template_db STRATEGY=<STRATEGY>;
+		ALTER DATABASE template_db IS_TEMPLATE = false;
+		DROP DATABASE dropme_db1;
+		DROP TABLE t;
+		DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
+		DROP TABLESPACE source_ts;
+		DROP DATABASE template_db;
+	];
+
+	$query =~ s/<STRATEGY>/$strategy/g;
+	$node_primary->safe_psql('postgres', $query);
+	$node_primary->wait_for_catchup($node_standby, 'replay',
+		$node_primary->lsn('write'));
+
+	# show "create missing directory" log message
+	$node_standby->safe_psql('postgres',
+		"ALTER SYSTEM SET log_min_messages TO debug1;");
+	$node_standby->stop('immediate');
+	# Should restart ignoring directory creation error.
+	is($node_standby->start(fail_ok => 1),
+		1, "standby node started for $strategy");
+	$node_standby->stop('immediate');
+}
+
+test_tablespace("FILE_COPY");
+test_tablespace("WAL_LOG");
+
+# Ensure that a missing tablespace directory during create database
+# replay immediately causes panic if the standby has already reached
+# consistent state (archive recovery is in progress).  This is
+# effective only for CREATE DATABASE WITH STRATEGY=FILE_COPY.
+
+my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
+$node_primary->init(allows_streaming => 1);
+$node_primary->start;
+
+# Create tablespace
+$node_primary->safe_psql(
+	'postgres', q[
+		SET allow_in_place_tablespaces=on;
+		CREATE TABLESPACE ts1 LOCATION ''
+			]);
+$node_primary->safe_psql('postgres',
+	"CREATE DATABASE db1 WITH TABLESPACE ts1 STRATEGY=FILE_COPY");
+
+# Take backup
+my $backup_name = 'my_backup';
+$node_primary->backup($backup_name);
+my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
+$node_standby->init_from_backup($node_primary, $backup_name,
+	has_streaming => 1);
+$node_standby->append_conf('postgresql.conf',
+	"allow_in_place_tablespaces = on");
+$node_standby->start;
+
+# Make sure standby reached consistency and starts accepting connections
+$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
+
+# Remove standby tablespace directory so it will be missing when
+# replay resumes.
+my $tspoid = $node_standby->safe_psql('postgres',
+	"SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
+my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
+File::Path::rmtree($tspdir);
+
+my $logstart = get_log_size($node_standby);
+
+# Create a database in the tablespace and a table in default tablespace
+$node_primary->safe_psql(
+	'postgres',
+	q[
+		CREATE TABLE should_not_replay_insertion(a int);
+		CREATE DATABASE db2 WITH TABLESPACE ts1 STRATEGY=FILE_COPY;
+		INSERT INTO should_not_replay_insertion VALUES (1);
+	]);
+
+# Standby should fail and should not silently skip replaying the wal
+# In this test, PANIC turns into WARNING by allow_in_place_tablespaces.
+# Check the log messages instead of confirming standby failure.
+my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
+while ($max_attempts-- >= 0)
+{
+	last
+	  if (
+		find_in_log(
+			$node_standby, "WARNING:  creating missing directory: pg_tblspc/",
+			$logstart));
+	sleep 1;
+}
+ok($max_attempts > 0, "invalid directory creation is detected");
+
+done_testing();
+
+
+# return the size of logfile of $node in bytes
+sub get_log_size
+{
+	my ($node) = @_;
+
+	return (stat $node->logfile)[7];
+}
+
+# find $pat in logfile of $node after $off-th byte
+sub find_in_log
+{
+	my ($node, $pat, $off) = @_;
+
+	$off = 0 unless defined $off;
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
+	return 0 if (length($log) <= $off);
+
+	$log = substr($log, $off);
+
+	return $log =~ m/$pat/;
+}
-- 
2.30.2