Checking pgwin32_is_junction() errors
Hi,
The comment for pgwin32_is_junction() says "Assumes the file exists,
so will return false if it doesn't (since a nonexistent file is not a
junction)". In fact that's the behaviour for any kind of error, and
although we set errno in that case, no caller ever checks it.
I think it'd be better to add missing_ok and elevel parameters,
following existing patterns. Unfortunately, it can't use the generic
frontend logging to implement elevel in frontend code from its current
location, because pgport can't call pgcommon. For now I came up with
a kludge to work around that problem, but I don't like it, and would
need to come up with something better...
Sketch code attached.
Attachments:
0001-Raise-errors-in-pgwin32_is_junction.patchtext/x-patch; charset=US-ASCII; name=0001-Raise-errors-in-pgwin32_is_junction.patchDownload
From 8e0e51359c0191cf673c3ddc87a3262b13f530a4 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Wed, 16 Mar 2022 23:05:39 +1300
Subject: [PATCH] Raise errors in pgwin32_is_junction().
XXX It's not very nice to use elevel like this in frontend code
---
src/backend/access/transam/xlog.c | 2 +-
src/backend/replication/basebackup.c | 4 ++--
src/backend/storage/file/fd.c | 2 +-
src/backend/utils/adt/misc.c | 2 +-
src/bin/pg_checksums/pg_checksums.c | 2 +-
src/bin/pg_rewind/file_ops.c | 2 +-
src/common/file_utils.c | 5 +++-
src/include/port.h | 2 +-
src/include/port/win32_port.h | 2 +-
src/port/dirmod.c | 36 +++++++++++++++++++++++++---
10 files changed, 46 insertions(+), 13 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 4ac3871c74..819b05177d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -8314,7 +8314,7 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
* directories directly under pg_tblspc, which would fail below.
*/
#ifdef WIN32
- if (!pgwin32_is_junction(fullpath))
+ if (!pgwin32_is_junction(fullpath, false, ERROR))
continue;
#else
if (get_dirent_type(fullpath, de, false, ERROR) != PGFILETYPE_LNK)
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 6884cad2c0..4be9090445 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -1352,7 +1352,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
#ifndef WIN32
S_ISLNK(statbuf.st_mode)
#else
- pgwin32_is_junction(pathbuf)
+ pgwin32_is_junction(pathbuf, true, ERROR)
#endif
)
{
@@ -1840,7 +1840,7 @@ convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
#ifndef WIN32
if (S_ISLNK(statbuf->st_mode))
#else
- if (pgwin32_is_junction(pathbuf))
+ if (pgwin32_is_junction(pathbuf, true, ERROR))
#endif
statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
}
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index 14b77f2861..ec2f49fefa 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -3453,7 +3453,7 @@ SyncDataDirectory(void)
xlog_is_symlink = true;
}
#else
- if (pgwin32_is_junction("pg_wal"))
+ if (pgwin32_is_junction("pg_wal", false, LOG))
xlog_is_symlink = true;
#endif
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index 89690be2ed..723504ab10 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -317,7 +317,7 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
* found, a relative path to the data directory is returned.
*/
#ifdef WIN32
- if (!pgwin32_is_junction(sourcepath))
+ if (!pgwin32_is_junction(sourcepath, false, ERROR))
PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
#else
if (lstat(sourcepath, &st) < 0)
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index 5f0f5ee62d..5720907d33 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -404,7 +404,7 @@ scan_directory(const char *basedir, const char *subdir, bool sizeonly)
#ifndef WIN32
else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
#else
- else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
+ else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn, false, 1))
#endif
{
/*
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 6cb288f099..1eda2baf2f 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -434,7 +434,7 @@ recurse_dir(const char *datadir, const char *parentpath,
#ifndef WIN32
else if (S_ISLNK(fst.st_mode))
#else
- else if (pgwin32_is_junction(fullpath))
+ else if (pgwin32_is_junction(fullpath, false, 1))
#endif
{
#if defined(HAVE_READLINK) || defined(WIN32)
diff --git a/src/common/file_utils.c b/src/common/file_utils.c
index 7138068633..2db909a3aa 100644
--- a/src/common/file_utils.c
+++ b/src/common/file_utils.c
@@ -88,8 +88,11 @@ fsync_pgdata(const char *pg_data,
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
+#elif defined(FRONTEND)
+ if (pgwin32_is_junction(pg_wal, false, 1))
+ xlog_is_symlink = true;
#else
- if (pgwin32_is_junction(pg_wal))
+ if (pgwin32_is_junction(pg_wal, false, ERROR))
xlog_is_symlink = true;
#endif
diff --git a/src/include/port.h b/src/include/port.h
index 3d103a2b31..b8118a3d51 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -284,7 +284,7 @@ extern int pgunlink(const char *path);
#if defined(WIN32) && !defined(__CYGWIN__)
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
+extern bool pgwin32_is_junction(const char *path, bool missing_ok, int elevel);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index d3cb765976..d9bec24348 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -230,7 +230,7 @@ int setitimer(int which, const struct itimerval *value, struct itimerval *oval
*/
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
+extern bool pgwin32_is_junction(const char *path, bool missing_ok, int elevel);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 7ce042e75d..127cb6a576 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -337,19 +337,49 @@ pgreadlink(const char *path, char *buf, size_t size)
}
/*
- * Assumes the file exists, so will return false if it doesn't
- * (since a nonexistent file is not a junction)
+ * In frontend code, elevel must be 0 or 1, where 1 means write an error and
+ * exit(1); in backend code it should be 0 or a level from elog.h, to pass to
+ * ereport(). If elevel is 0, or missing_ok is set and the path doesn't exist,
+ * no logging is done. The caller should check errno on false return, unless
+ * elevel is set to a level that doesn't return on error.
*/
bool
-pgwin32_is_junction(const char *path)
+pgwin32_is_junction(const char *path, bool missing_ok, int elevel)
{
DWORD attr = GetFileAttributes(path);
+#ifdef FRONTEND
+ /*
+ * We can't use ereport, and libport can't use the logging from libcommon
+ * due to library order, so the options for logging are limited.
+ */
+ Assert(elevel == 0 || elevel == 1);
+#endif
+
if (attr == INVALID_FILE_ATTRIBUTES)
{
_dosmaperr(GetLastError());
+
+ if (errno == ENOENT && missing_ok)
+ return false;
+
+ if (elevel != 0)
+ {
+#ifndef FRONTEND
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not get attributes for file \"%s\": %m",
+ path)));
+#else
+ int save_errno = errno;
+ fprintf(stderr, _("could not get attributes for file \"%s\": %s\n"),
+ path, strerror(save_errno));
+ exit(1);
+#endif
+ }
return false;
}
+ errno = 0;
return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT);
}
#endif /* defined(WIN32) && !defined(__CYGWIN__) */
--
2.35.1
On Thu, Mar 24, 2022 at 04:30:26PM +1300, Thomas Munro wrote:
I think it'd be better to add missing_ok and elevel parameters,
following existing patterns. Unfortunately, it can't use the generic
frontend logging to implement elevel in frontend code from its current
location, because pgport can't call pgcommon. For now I came up with
a kludge to work around that problem, but I don't like it, and would
need to come up with something better...
The only barrier reason why elevel if needed is because of pg_wal in
SyncDataDirectory() that cannot fail hard. I don't have a great idea
here, except using a bits32 with some bitwise flags to control the
behavior of the routine, aka something close to a MISSING_OK and a
FAIL_HARD_ON_ERROR. This pattern exists already in some of the
*Extended() routines.
--
Michael
Here's a better idea, now that I'm emboldened by having working CI for
Windows frankenbuilds, and since I broke some stuff in this area on
MSYS[1]/messages/by-id/b9ddf605-6b36-f90d-7c30-7b3e95c46276@dunslane.net, which caused me to look more closely at this area.
Why don't we just nuke pgwin32_is_junction() from orbit, and teach
Windows how to lstat()? We're already defining our own replacement
stat() used in both MSVC and MSYS builds along with our own junction
point-based symlink() and readlink() functions, and lstat() was
already suggested in a comment in win32stat.c.
There's one curious change in the draft patch attached: you can't
unlink() a junction point, you have to rmdir() it. Previously, things
that traverse directories without ever calling pgwin32_is_junction()
would see junction points as S_ISDIR() and call rmdir(), which was OK,
but now they see S_ISLNK() and call unlink(). So I taught unlink() to
try both things. Which is kinda weird, and not beautiful, especially
when combined with the existing looping weirdness.
0001 is a copy of v2 of Melih Mutlu's CI patch[2]/messages/by-id/CAGPVpCSKS9E0An4=e7ZDnme+y=WOcQFJYJegKO8kE9=gh8NJKQ@mail.gmail.com to show cfbot how to
test this on MSYS (alongside the normal MSVC result), but that's not
part of this submission.
[1]: /messages/by-id/b9ddf605-6b36-f90d-7c30-7b3e95c46276@dunslane.net
[2]: /messages/by-id/CAGPVpCSKS9E0An4=e7ZDnme+y=WOcQFJYJegKO8kE9=gh8NJKQ@mail.gmail.com
Attachments:
0001-Added-Windows-with-MinGW-environment-in-Cirrus-CI.patchtext/x-patch; charset=US-ASCII; name=0001-Added-Windows-with-MinGW-environment-in-Cirrus-CI.patchDownload
From ec732571a1fa88fefcf9834987114716eb9c5998 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <memutlu@microsoft.com>
Date: Mon, 21 Feb 2022 14:46:05 +0300
Subject: [PATCH 1/3] Added Windows with MinGW environment in Cirrus CI
---
.cirrus.yml | 79 +++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 65 insertions(+), 14 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index f23d6cae55..3d8c4cd813 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -23,7 +23,6 @@ env:
CHECKFLAGS: -Otarget
PROVE_FLAGS: --timer
PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
- TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
PG_TEST_EXTRA: kerberos ldap ssl
@@ -338,13 +337,30 @@ task:
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
+WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
+ env:
+ # Half the allowed per-user CPU cores
+ CPUS: 4
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
+
+ # Avoids port conflicts between concurrent tap test runs
+ PG_TEST_USE_UNIX_SOCKETS: 1
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+
+ sysinfo_script: |
+ chcp
+ systeminfo
+ powershell -Command get-psdrive -psprovider filesystem
+ set
+
task:
+ << : *WINDOWS_ENVIRONMENT_BASE
name: Windows - Server 2019, VS 2019
env:
- # Half the allowed per-user CPU cores
- CPUS: 4
-
# Our windows infrastructure doesn't have test concurrency above the level
# of a single vcregress test target. Due to that, it's useful to run prove
# with multiple jobs. For the other tasks it isn't, because two sources
@@ -354,15 +370,11 @@ task:
# likely can be improved upon further.
PROVE_FLAGS: -j10 --timer
- # The default cirrus working dir is in a directory msbuild complains about
- CIRRUS_WORKING_DIR: "c:/cirrus"
# Avoid re-installing over and over
NO_TEMP_INSTALL: 1
# git's tar doesn't deal with drive letters, see
# https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
TAR: "c:/windows/system32/tar.exe"
- # Avoids port conflicts between concurrent tap test runs
- PG_TEST_USE_UNIX_SOCKETS: 1
PG_REGRESS_SOCK_DIR: "c:/cirrus/"
# -m enables parallelism
# verbosity:minimal + Summary reduce verbosity, while keeping a summary of
@@ -393,12 +405,6 @@ task:
cpu: $CPUS
memory: 4G
- sysinfo_script: |
- chcp
- systeminfo
- powershell -Command get-psdrive -psprovider filesystem
- set
-
setup_additional_packages_script: |
REM choco install -y --no-progress ...
@@ -456,6 +462,51 @@ task:
path: "crashlog-*.txt"
type: text/plain
+task:
+ << : *WINDOWS_ENVIRONMENT_BASE
+ name: Windows - Server 2019, MinGW64
+ windows_container:
+ image: $CONTAINER_REPO/windows_ci_mingw64:latest
+ cpu: $CPUS
+ memory: 4G
+ env:
+ CCACHE_DIR: C:/msys64/ccache
+ BUILD_DIR: "%CIRRUS_WORKING_DIR%/build"
+
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+
+ mingw_info_script:
+ - C:\msys64\usr\bin\bash.exe -lc "where gcc"
+ - C:\msys64\usr\bin\bash.exe -lc "gcc --version"
+ - C:\msys64\usr\bin\bash.exe -lc "where perl"
+ - C:\msys64\usr\bin\bash.exe -lc "perl --version"
+
+ configure_script:
+ - C:\msys64\usr\bin\bash.exe -lc "mkdir %BUILD_DIR% &&
+ cd %BUILD_DIR% &&
+ %CIRRUS_WORKING_DIR%/configure
+ --host=x86_64-w64-mingw32
+ --enable-cassert
+ --enable-tap-tests
+ --with-icu
+ --with-libxml
+ --with-libxslt
+ --with-lz4
+ --enable-debug
+ CC='ccache gcc'
+ CXX='ccache g++'"
+
+ build_script:
+ C:\msys64\usr\bin\bash.exe -lc "cd %BUILD_DIR% && make -s world-bin -j${CPUS}"
+
+ upload_caches: ccache
+
+ tests_script:
+ - set "NoDefaultCurrentDirectoryInExePath=0"
+ - C:\msys64\usr\bin\bash.exe -lc "cd %BUILD_DIR% && make -s ${CHECK} ${CHECKFLAGS} -j${CPUS} TMPDIR=%BUILD_DIR%/tmp_install"
+
+ on_failure: *on_failure
task:
name: CompilerWarnings
--
2.36.1
0002-Provide-lstat-for-Windows.patchtext/x-patch; charset=US-ASCII; name=0002-Provide-lstat-for-Windows.patchDownload
From 2390efc8ffebcb975c0c96b63479a1f22bf8e422 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Thu, 28 Jul 2022 15:19:38 +1200
Subject: [PATCH 2/3] Provide lstat() for Windows.
Junction points will be reported with S_ISLNK(x.st_mode), like in POSIX.
stat() will follow symlinks, like in POSIX (but only one level before it
gives up, unlike in POSIX).
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/include/port/win32_port.h | 18 +++++++-
src/port/win32stat.c | 79 +++++++++++++++++++++++++++++++++--
2 files changed, 92 insertions(+), 5 deletions(-)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index 4de5bf3bf6..b8cf2e1480 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -286,10 +286,11 @@ struct stat /* This should match struct __stat64 */
extern int _pgfstat64(int fileno, struct stat *buf);
extern int _pgstat64(const char *name, struct stat *buf);
+extern int _pglstat64(const char *name, struct stat *buf);
#define fstat(fileno, sb) _pgfstat64(fileno, sb)
#define stat(path, sb) _pgstat64(path, sb)
-#define lstat(path, sb) _pgstat64(path, sb)
+#define lstat(path, sb) _pglstat64(path, sb)
/* These macros are not provided by older MinGW, nor by MSVC */
#ifndef S_IRUSR
@@ -335,6 +336,21 @@ extern int _pgstat64(const char *name, struct stat *buf);
#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
#endif
+/*
+ * In order for lstat() to be able to report junction points as symlinks, we
+ * need to hijack a bit in st_mode, since neither MSVC nor MinGW provides
+ * S_ISLNK and there aren't any spare bits. We'll steal the one for character
+ * devices, because we don't otherwise make use of those.
+ */
+#ifdef S_ISLNK
+#error "S_ISLNK is already defined"
+#endif
+#ifdef S_IFLNK
+#error "S_IFLNK is already defined"
+#endif
+#define S_IFLNK S_IFCHR
+#define S_ISLNK(m) (((m) & S_IFLNK) == S_IFLNK)
+
/*
* Supplement to <fcntl.h>.
* This is the same value as _O_NOINHERIT in the MS header file. This is
diff --git a/src/port/win32stat.c b/src/port/win32stat.c
index e03ed5f35c..686c510fed 100644
--- a/src/port/win32stat.c
+++ b/src/port/win32stat.c
@@ -15,7 +15,11 @@
#ifdef WIN32
+#define UMDF_USING_NTSTATUS
+
#include "c.h"
+#include "port/win32ntdll.h"
+
#include <windows.h>
/*
@@ -107,12 +111,10 @@ fileinfo_to_stat(HANDLE hFile, struct stat *buf)
}
/*
- * Windows implementation of stat().
- *
- * This currently also implements lstat(), though perhaps that should change.
+ * Windows implementation of lstat().
*/
int
-_pgstat64(const char *name, struct stat *buf)
+_pglstat64(const char *name, struct stat *buf)
{
/*
* Our open wrapper will report STATUS_DELETE_PENDING as ENOENT. We
@@ -129,10 +131,79 @@ _pgstat64(const char *name, struct stat *buf)
ret = fileinfo_to_stat(hFile, buf);
+ /*
+ * Unfortunately it's not possible for fileinfo_to_stat() to see a
+ * FILE_ATTRIBUTE_REPARSE_POINT flag with just a handle, so at this point
+ * junction points appear as directories. Ask for the full attributes
+ * while we still have the handle open. Someone else can unlink the file
+ * while we have it open, but they can't create another file with the same
+ * name.
+ */
+ if (ret == 0 && S_ISDIR(buf->st_mode))
+ {
+ DWORD attr;
+
+ attr = GetFileAttributes(name);
+ if (attr == INVALID_FILE_ATTRIBUTES)
+ {
+ DWORD err;
+
+ /* If it's been unlinked since we opened it, report as ENOENT. */
+ err = GetLastError();
+ if (err == ERROR_ACCESS_DENIED &&
+ pg_RtlGetLastNtStatus() == STATUS_DELETE_PENDING)
+ errno = ENOENT;
+ else
+ _dosmaperr(err);
+
+ ret = -1;
+ }
+ else if (attr & FILE_ATTRIBUTE_REPARSE_POINT)
+ {
+ buf->st_mode &= ~S_IFDIR;
+ buf->st_mode |= S_IFLNK;
+ }
+ }
+
CloseHandle(hFile);
return ret;
}
+/*
+ * Windows implementation of stat().
+ */
+int
+_pgstat64(const char *name, struct stat *buf)
+{
+ int ret;
+
+ ret = _pglstat64(name, buf);
+
+ /* Do we need to follow a symlink (junction point)? */
+ if (ret == 0 && S_ISLNK(buf->st_mode))
+ {
+ char next[MAXPGPATH];
+
+ if (readlink(name, next, sizeof(next)) < 0)
+ return -1;
+
+ ret = _pglstat64(next, buf);
+ if (ret == 0 && S_ISLNK(buf->st_mode))
+ {
+ /*
+ * We're only prepared to go one hop, because we only expect to
+ * deal with the simple cases that we create. The error for too
+ * many symlinks is supposed to be ELOOP, but Windows hasn't got
+ * it.
+ */
+ errno = EIO;
+ return -1;
+ }
+ }
+
+ return ret;
+}
+
/*
* Windows implementation of fstat().
*/
--
2.36.1
0003-Remove-pgwin32_is_junction.patchtext/x-patch; charset=US-ASCII; name=0003-Remove-pgwin32_is_junction.patchDownload
From 2d44c134f62872889439db6910709af745901dd9 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Thu, 28 Jul 2022 15:25:10 +1200
Subject: [PATCH 3/3] Remove pgwin32_is_junction().
Now that lstat() reports junction points with S_IFLNK, there is no need
for sparate code paths for Windows in a few places. The coding around
pgwin32_is_junction() was a bit suspect anyway, as we never checked for
errors.
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/backend/commands/tablespace.c | 3 +--
src/backend/replication/basebackup.c | 12 +-----------
src/backend/storage/file/fd.c | 5 -----
src/backend/utils/adt/misc.c | 7 -------
src/bin/pg_checksums/pg_checksums.c | 4 ----
src/bin/pg_rewind/file_ops.c | 4 ----
src/common/file_utils.c | 21 --------------------
src/include/port.h | 1 -
src/include/port/win32_port.h | 1 -
src/port/dirmod.c | 29 +++++++++++++---------------
10 files changed, 15 insertions(+), 72 deletions(-)
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index cb7d46089a..b74f40aac6 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -827,8 +827,7 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo)
/*
* Try to remove the symlink. We must however deal with the possibility
* that it's a directory instead of a symlink --- this could happen during
- * WAL replay (see TablespaceCreateDbspace), and it is also the case on
- * Windows where junction points lstat() as directories.
+ * WAL replay (see TablespaceCreateDbspace).
*
* Note: in the redo case, we'll return true if this final step fails;
* there's no point in retrying it. Also, ENOENT should provoke no more
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 637c0ce459..638de07035 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -1320,13 +1320,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
}
/* Allow symbolic links in pg_tblspc only */
- if (strcmp(path, "./pg_tblspc") == 0 &&
-#ifndef WIN32
- S_ISLNK(statbuf.st_mode)
-#else
- pgwin32_is_junction(pathbuf)
-#endif
- )
+ if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
{
#if defined(HAVE_READLINK) || defined(WIN32)
char linkpath[MAXPGPATH];
@@ -1809,11 +1803,7 @@ static void
convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
{
/* If symlink, write it as a directory anyway */
-#ifndef WIN32
if (S_ISLNK(statbuf->st_mode))
-#else
- if (pgwin32_is_junction(pathbuf))
-#endif
statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
}
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index f904f60c08..566ebe26a0 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -3377,7 +3377,6 @@ SyncDataDirectory(void)
*/
xlog_is_symlink = false;
-#ifndef WIN32
{
struct stat st;
@@ -3389,10 +3388,6 @@ SyncDataDirectory(void)
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
-#else
- if (pgwin32_is_junction("pg_wal"))
- xlog_is_symlink = true;
-#endif
#ifdef HAVE_SYNCFS
if (recovery_init_sync_method == RECOVERY_INIT_SYNC_METHOD_SYNCFS)
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index 89690be2ed..412039ea6b 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -283,9 +283,7 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
char sourcepath[MAXPGPATH];
char targetpath[MAXPGPATH];
int rllen;
-#ifndef WIN32
struct stat st;
-#endif
/*
* It's useful to apply this function to pg_class.reltablespace, wherein
@@ -316,10 +314,6 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
* created with allow_in_place_tablespaces enabled. If a directory is
* found, a relative path to the data directory is returned.
*/
-#ifdef WIN32
- if (!pgwin32_is_junction(sourcepath))
- PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
-#else
if (lstat(sourcepath, &st) < 0)
{
ereport(ERROR,
@@ -330,7 +324,6 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
if (!S_ISLNK(st.st_mode))
PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
-#endif
/*
* In presence of a link or a junction point, return the path pointing to.
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index dc20122c89..324ccf7783 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -384,11 +384,7 @@ scan_directory(const char *basedir, const char *subdir, bool sizeonly)
if (!sizeonly)
scan_file(fn, segmentno);
}
-#ifndef WIN32
else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
-#else
- else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
-#endif
{
/*
* If going through the entries of pg_tblspc, we assume to operate
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 6cb288f099..7d98283c79 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -431,11 +431,7 @@ recurse_dir(const char *datadir, const char *parentpath,
/* recurse to handle subdirectories */
recurse_dir(datadir, path, callback);
}
-#ifndef WIN32
else if (S_ISLNK(fst.st_mode))
-#else
- else if (pgwin32_is_junction(fullpath))
-#endif
{
#if defined(HAVE_READLINK) || defined(WIN32)
char link_target[MAXPGPATH];
diff --git a/src/common/file_utils.c b/src/common/file_utils.c
index 966b987d64..71c232d983 100644
--- a/src/common/file_utils.c
+++ b/src/common/file_utils.c
@@ -79,7 +79,6 @@ fsync_pgdata(const char *pg_data,
*/
xlog_is_symlink = false;
-#ifndef WIN32
{
struct stat st;
@@ -88,10 +87,6 @@ fsync_pgdata(const char *pg_data,
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
-#else
- if (pgwin32_is_junction(pg_wal))
- xlog_is_symlink = true;
-#endif
/*
* If possible, hint to the kernel that we're soon going to fsync the data
@@ -465,21 +460,5 @@ get_dirent_type(const char *path,
#endif
}
-#if defined(WIN32) && !defined(_MSC_VER)
-
- /*
- * If we're on native Windows (not Cygwin, which has its own POSIX
- * symlinks), but not using the MSVC compiler, then we're using a
- * readdir() emulation provided by the MinGW runtime that has no d_type.
- * Since the lstat() fallback code reports junction points as directories,
- * we need an extra system call to check if we should report them as
- * symlinks instead, following our convention.
- */
- if (result == PGFILETYPE_DIR &&
- !look_through_symlinks &&
- pgwin32_is_junction(path))
- result = PGFILETYPE_LNK;
-#endif
-
return result;
}
diff --git a/src/include/port.h b/src/include/port.h
index d39b04141f..4b978f61bd 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -284,7 +284,6 @@ extern int pgunlink(const char *path);
#if defined(WIN32) && !defined(__CYGWIN__)
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index b8cf2e1480..72a0dd572e 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -238,7 +238,6 @@ int setitimer(int which, const struct itimerval *value, struct itimerval *oval
*/
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 7ce042e75d..01f9d1362b 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -111,6 +111,19 @@ pgunlink(const char *path)
{
if (errno != EACCES)
return -1;
+
+ /*
+ * Also try with rmdir(), because that's how you remove junction
+ * points. Since we pretend junction points are symlinks, various code
+ * might try to call unlink(). We'll preserve unlink()'s errno though;
+ * if we got ENOENT then unlink() will presumably return the same next
+ * time around, ENOTEMPTY shouldn't be possible for a junction point,
+ * and EACCES is what unlink() returned.
+ */
+ if (rmdir(path) == 0)
+ break;
+ errno = EACCES;
+
if (++loops > 100) /* time out after 10 sec */
return -1;
pg_usleep(100000); /* us */
@@ -336,20 +349,4 @@ pgreadlink(const char *path, char *buf, size_t size)
return r;
}
-/*
- * Assumes the file exists, so will return false if it doesn't
- * (since a nonexistent file is not a junction)
- */
-bool
-pgwin32_is_junction(const char *path)
-{
- DWORD attr = GetFileAttributes(path);
-
- if (attr == INVALID_FILE_ATTRIBUTES)
- {
- _dosmaperr(GetLastError());
- return false;
- }
- return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT);
-}
#endif /* defined(WIN32) && !defined(__CYGWIN__) */
--
2.36.1
On Thu, Jul 28, 2022 at 9:31 PM Thomas Munro <thomas.munro@gmail.com> wrote:
There's one curious change in the draft patch attached: you can't
unlink() a junction point, you have to rmdir() it. Previously, things
that traverse directories without ever calling pgwin32_is_junction()
would see junction points as S_ISDIR() and call rmdir(), which was OK,
but now they see S_ISLNK() and call unlink(). So I taught unlink() to
try both things. Which is kinda weird, and not beautiful, especially
when combined with the existing looping weirdness.
Here's a new attempt at unlink(), this time in its own patch. This
version is a little more careful about calling rmdir() only after
checking that it is a junction point, so that unlink("a directory")
fails just like on Unix (well, POSIX says that that should fail with
EPERM, not EACCES, and implementations are allowed to make it work
anyway, but it doesn't seem helpful to allow it to work there when
every OS I know of fails with EPERM or EISDIR). That check is racy,
but should be good enough for our purposes, no (see comment for a note
on that)?
Longer term, I wonder if we should get rid of our use of symlinks, and
instead just put paths in a file and do our own path translation. But
for now, this patch set completes the set of junction point-based
emulations, and, IMHO, cleans up a confusing aspect of our code.
As before, 0001 is just for cfbot to add an MSYS checkmark.
Attachments:
v3-0001-Added-Windows-with-MinGW-environment-in-Cirrus-CI.patchtext/x-patch; charset=US-ASCII; name=v3-0001-Added-Windows-with-MinGW-environment-in-Cirrus-CI.patchDownload
From c89ee0d78944e05fa28c1d6c2939d94b6e39987a Mon Sep 17 00:00:00 2001
From: Melih Mutlu <memutlu@microsoft.com>
Date: Mon, 21 Feb 2022 14:46:05 +0300
Subject: [PATCH v3 1/4] Added Windows with MinGW environment in Cirrus CI
---
.cirrus.yml | 79 +++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 65 insertions(+), 14 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index 72735d225a..8decef68e8 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -23,7 +23,6 @@ env:
CHECKFLAGS: -Otarget
PROVE_FLAGS: --timer
PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
- TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
PG_TEST_EXTRA: kerberos ldap ssl
@@ -338,13 +337,30 @@ task:
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
+WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
+ env:
+ # Half the allowed per-user CPU cores
+ CPUS: 4
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
+
+ # Avoids port conflicts between concurrent tap test runs
+ PG_TEST_USE_UNIX_SOCKETS: 1
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+
+ sysinfo_script: |
+ chcp
+ systeminfo
+ powershell -Command get-psdrive -psprovider filesystem
+ set
+
task:
+ << : *WINDOWS_ENVIRONMENT_BASE
name: Windows - Server 2019, VS 2019
env:
- # Half the allowed per-user CPU cores
- CPUS: 4
-
# Our windows infrastructure doesn't have test concurrency above the level
# of a single vcregress test target. Due to that, it's useful to run prove
# with multiple jobs. For the other tasks it isn't, because two sources
@@ -354,15 +370,11 @@ task:
# likely can be improved upon further.
PROVE_FLAGS: -j10 --timer
- # The default cirrus working dir is in a directory msbuild complains about
- CIRRUS_WORKING_DIR: "c:/cirrus"
# Avoid re-installing over and over
NO_TEMP_INSTALL: 1
# git's tar doesn't deal with drive letters, see
# https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
TAR: "c:/windows/system32/tar.exe"
- # Avoids port conflicts between concurrent tap test runs
- PG_TEST_USE_UNIX_SOCKETS: 1
PG_REGRESS_SOCK_DIR: "c:/cirrus/"
# -m enables parallelism
# verbosity:minimal + Summary reduce verbosity, while keeping a summary of
@@ -393,12 +405,6 @@ task:
cpu: $CPUS
memory: 4G
- sysinfo_script: |
- chcp
- systeminfo
- powershell -Command get-psdrive -psprovider filesystem
- set
-
setup_additional_packages_script: |
REM choco install -y --no-progress ...
@@ -456,6 +462,51 @@ task:
path: "crashlog-*.txt"
type: text/plain
+task:
+ << : *WINDOWS_ENVIRONMENT_BASE
+ name: Windows - Server 2019, MinGW64
+ windows_container:
+ image: $CONTAINER_REPO/windows_ci_mingw64:latest
+ cpu: $CPUS
+ memory: 4G
+ env:
+ CCACHE_DIR: C:/msys64/ccache
+ BUILD_DIR: "%CIRRUS_WORKING_DIR%/build"
+
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+
+ mingw_info_script:
+ - C:\msys64\usr\bin\bash.exe -lc "where gcc"
+ - C:\msys64\usr\bin\bash.exe -lc "gcc --version"
+ - C:\msys64\usr\bin\bash.exe -lc "where perl"
+ - C:\msys64\usr\bin\bash.exe -lc "perl --version"
+
+ configure_script:
+ - C:\msys64\usr\bin\bash.exe -lc "mkdir %BUILD_DIR% &&
+ cd %BUILD_DIR% &&
+ %CIRRUS_WORKING_DIR%/configure
+ --host=x86_64-w64-mingw32
+ --enable-cassert
+ --enable-tap-tests
+ --with-icu
+ --with-libxml
+ --with-libxslt
+ --with-lz4
+ --enable-debug
+ CC='ccache gcc'
+ CXX='ccache g++'"
+
+ build_script:
+ C:\msys64\usr\bin\bash.exe -lc "cd %BUILD_DIR% && make -s world-bin -j${CPUS}"
+
+ upload_caches: ccache
+
+ tests_script:
+ - set "NoDefaultCurrentDirectoryInExePath=0"
+ - C:\msys64\usr\bin\bash.exe -lc "cd %BUILD_DIR% && make -s ${CHECK} ${CHECKFLAGS} -j${CPUS} TMPDIR=%BUILD_DIR%/tmp_install"
+
+ on_failure: *on_failure
task:
name: CompilerWarnings
--
2.37.1
v3-0002-Provide-lstat-for-Windows.patchtext/x-patch; charset=US-ASCII; name=v3-0002-Provide-lstat-for-Windows.patchDownload
From 5e64700facc43fd937349bc8d843b15abb8da5cc Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Thu, 28 Jul 2022 15:19:38 +1200
Subject: [PATCH v3 2/4] Provide lstat() for Windows.
Junction points will be reported with S_ISLNK(x.st_mode), like in POSIX.
stat() will follow symlinks, like in POSIX (but only one level before it
gives up, unlike in POSIX).
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/include/port/win32_port.h | 18 +++++++-
src/port/win32stat.c | 79 +++++++++++++++++++++++++++++++++--
2 files changed, 92 insertions(+), 5 deletions(-)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index 4de5bf3bf6..b8cf2e1480 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -286,10 +286,11 @@ struct stat /* This should match struct __stat64 */
extern int _pgfstat64(int fileno, struct stat *buf);
extern int _pgstat64(const char *name, struct stat *buf);
+extern int _pglstat64(const char *name, struct stat *buf);
#define fstat(fileno, sb) _pgfstat64(fileno, sb)
#define stat(path, sb) _pgstat64(path, sb)
-#define lstat(path, sb) _pgstat64(path, sb)
+#define lstat(path, sb) _pglstat64(path, sb)
/* These macros are not provided by older MinGW, nor by MSVC */
#ifndef S_IRUSR
@@ -335,6 +336,21 @@ extern int _pgstat64(const char *name, struct stat *buf);
#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
#endif
+/*
+ * In order for lstat() to be able to report junction points as symlinks, we
+ * need to hijack a bit in st_mode, since neither MSVC nor MinGW provides
+ * S_ISLNK and there aren't any spare bits. We'll steal the one for character
+ * devices, because we don't otherwise make use of those.
+ */
+#ifdef S_ISLNK
+#error "S_ISLNK is already defined"
+#endif
+#ifdef S_IFLNK
+#error "S_IFLNK is already defined"
+#endif
+#define S_IFLNK S_IFCHR
+#define S_ISLNK(m) (((m) & S_IFLNK) == S_IFLNK)
+
/*
* Supplement to <fcntl.h>.
* This is the same value as _O_NOINHERIT in the MS header file. This is
diff --git a/src/port/win32stat.c b/src/port/win32stat.c
index e03ed5f35c..686c510fed 100644
--- a/src/port/win32stat.c
+++ b/src/port/win32stat.c
@@ -15,7 +15,11 @@
#ifdef WIN32
+#define UMDF_USING_NTSTATUS
+
#include "c.h"
+#include "port/win32ntdll.h"
+
#include <windows.h>
/*
@@ -107,12 +111,10 @@ fileinfo_to_stat(HANDLE hFile, struct stat *buf)
}
/*
- * Windows implementation of stat().
- *
- * This currently also implements lstat(), though perhaps that should change.
+ * Windows implementation of lstat().
*/
int
-_pgstat64(const char *name, struct stat *buf)
+_pglstat64(const char *name, struct stat *buf)
{
/*
* Our open wrapper will report STATUS_DELETE_PENDING as ENOENT. We
@@ -129,10 +131,79 @@ _pgstat64(const char *name, struct stat *buf)
ret = fileinfo_to_stat(hFile, buf);
+ /*
+ * Unfortunately it's not possible for fileinfo_to_stat() to see a
+ * FILE_ATTRIBUTE_REPARSE_POINT flag with just a handle, so at this point
+ * junction points appear as directories. Ask for the full attributes
+ * while we still have the handle open. Someone else can unlink the file
+ * while we have it open, but they can't create another file with the same
+ * name.
+ */
+ if (ret == 0 && S_ISDIR(buf->st_mode))
+ {
+ DWORD attr;
+
+ attr = GetFileAttributes(name);
+ if (attr == INVALID_FILE_ATTRIBUTES)
+ {
+ DWORD err;
+
+ /* If it's been unlinked since we opened it, report as ENOENT. */
+ err = GetLastError();
+ if (err == ERROR_ACCESS_DENIED &&
+ pg_RtlGetLastNtStatus() == STATUS_DELETE_PENDING)
+ errno = ENOENT;
+ else
+ _dosmaperr(err);
+
+ ret = -1;
+ }
+ else if (attr & FILE_ATTRIBUTE_REPARSE_POINT)
+ {
+ buf->st_mode &= ~S_IFDIR;
+ buf->st_mode |= S_IFLNK;
+ }
+ }
+
CloseHandle(hFile);
return ret;
}
+/*
+ * Windows implementation of stat().
+ */
+int
+_pgstat64(const char *name, struct stat *buf)
+{
+ int ret;
+
+ ret = _pglstat64(name, buf);
+
+ /* Do we need to follow a symlink (junction point)? */
+ if (ret == 0 && S_ISLNK(buf->st_mode))
+ {
+ char next[MAXPGPATH];
+
+ if (readlink(name, next, sizeof(next)) < 0)
+ return -1;
+
+ ret = _pglstat64(next, buf);
+ if (ret == 0 && S_ISLNK(buf->st_mode))
+ {
+ /*
+ * We're only prepared to go one hop, because we only expect to
+ * deal with the simple cases that we create. The error for too
+ * many symlinks is supposed to be ELOOP, but Windows hasn't got
+ * it.
+ */
+ errno = EIO;
+ return -1;
+ }
+ }
+
+ return ret;
+}
+
/*
* Windows implementation of fstat().
*/
--
2.37.1
v3-0003-Make-unlink-work-for-junction-points-on-Windows.patchtext/x-patch; charset=US-ASCII; name=v3-0003-Make-unlink-work-for-junction-points-on-Windows.patchDownload
From 77925ac5fa68b0915bd309d977c85ecf1128f9fc Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Mon, 1 Aug 2022 15:43:13 +1200
Subject: [PATCH v3 3/4] Make unlink() work for junction points on Windows.
To support harmonization of Windows and Unix code, teach our unlink()
wrapper that junction points need to be unlinked with rmdir() on
Windows.
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/port/dirmod.c | 28 +++++++++++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 7ce042e75d..ea191e99c6 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -99,6 +99,32 @@ int
pgunlink(const char *path)
{
int loops = 0;
+ struct stat st;
+
+ /*
+ * This function might be called for a regular file or for a junction
+ * point (which we use to emulate symlinks). The latter must be unlinked
+ * with rmdir() on Windows. Before we worry about any of that, let's see
+ * if we can unlink directly, since that's expected to be the most common
+ * case.
+ */
+ if (unlink(path) == 0)
+ return 0;
+ if (errno != EACCES)
+ return -1;
+
+ /*
+ * EACCES is reported for many reasons including unlink() of a junction
+ * point. Check if that's the case so we can redirect to rmdir().
+ *
+ * Note that by checking only once, we can't cope with a path that changes
+ * from regular file to junction point underneath us while we're retrying
+ * due to sharing violations, but that seems unlikely. We could perhaps
+ * prevent that by holding a file handle ourselves across the lstat() and
+ * the retry loop, but that seems like over-engineering for now.
+ */
+ if (lstat(path, &st) < 0)
+ return -1;
/*
* We need to loop because even though PostgreSQL uses flags that allow
@@ -107,7 +133,7 @@ pgunlink(const char *path)
* someone else to close the file, as the caller might be holding locks
* and blocking other backends.
*/
- while (unlink(path))
+ while ((S_ISLNK(st.st_mode) ? rmdir(path) : unlink(path)) < 0)
{
if (errno != EACCES)
return -1;
--
2.37.1
v3-0004-Replace-pgwin32_is_junction-with-lstat.patchtext/x-patch; charset=US-ASCII; name=v3-0004-Replace-pgwin32_is_junction-with-lstat.patchDownload
From 4391644be135e15ffb052594c4ae54c4d83569df Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Thu, 28 Jul 2022 15:25:10 +1200
Subject: [PATCH v3 4/4] Replace pgwin32_is_junction() with lstat().
Now that lstat() reports junction points with S_IFLNK and unlink() can
unlink them, there is no need for separate code paths for Windows in a
few places.
The coding around pgwin32_is_junction() was a bit suspect anyway, as we
never checked for errors, and we also know that errors can be spuriously
reported because of transient sharing violations on this OS. The
lstat()-based code tries to deal with that by retrying.
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/backend/commands/tablespace.c | 3 +--
src/backend/replication/basebackup.c | 12 +-----------
src/backend/storage/file/fd.c | 5 -----
src/backend/utils/adt/misc.c | 7 -------
src/bin/pg_checksums/pg_checksums.c | 4 ----
src/bin/pg_rewind/file_ops.c | 4 ----
src/common/file_utils.c | 21 ---------------------
src/include/port.h | 1 -
src/include/port/win32_port.h | 1 -
src/port/dirmod.c | 16 ----------------
10 files changed, 2 insertions(+), 72 deletions(-)
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 570ce3dbd5..dc0d7befd9 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -805,8 +805,7 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo)
/*
* Try to remove the symlink. We must however deal with the possibility
* that it's a directory instead of a symlink --- this could happen during
- * WAL replay (see TablespaceCreateDbspace), and it is also the case on
- * Windows where junction points lstat() as directories.
+ * WAL replay (see TablespaceCreateDbspace).
*
* Note: in the redo case, we'll return true if this final step fails;
* there's no point in retrying it. Also, ENOENT should provoke no more
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 7f85071229..a048cd5f31 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -1322,13 +1322,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
}
/* Allow symbolic links in pg_tblspc only */
- if (strcmp(path, "./pg_tblspc") == 0 &&
-#ifndef WIN32
- S_ISLNK(statbuf.st_mode)
-#else
- pgwin32_is_junction(pathbuf)
-#endif
- )
+ if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
{
#if defined(HAVE_READLINK) || defined(WIN32)
char linkpath[MAXPGPATH];
@@ -1811,11 +1805,7 @@ static void
convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
{
/* If symlink, write it as a directory anyway */
-#ifndef WIN32
if (S_ISLNK(statbuf->st_mode))
-#else
- if (pgwin32_is_junction(pathbuf))
-#endif
statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
}
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index f904f60c08..566ebe26a0 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -3377,7 +3377,6 @@ SyncDataDirectory(void)
*/
xlog_is_symlink = false;
-#ifndef WIN32
{
struct stat st;
@@ -3389,10 +3388,6 @@ SyncDataDirectory(void)
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
-#else
- if (pgwin32_is_junction("pg_wal"))
- xlog_is_symlink = true;
-#endif
#ifdef HAVE_SYNCFS
if (recovery_init_sync_method == RECOVERY_INIT_SYNC_METHOD_SYNCFS)
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index 89690be2ed..412039ea6b 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -283,9 +283,7 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
char sourcepath[MAXPGPATH];
char targetpath[MAXPGPATH];
int rllen;
-#ifndef WIN32
struct stat st;
-#endif
/*
* It's useful to apply this function to pg_class.reltablespace, wherein
@@ -316,10 +314,6 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
* created with allow_in_place_tablespaces enabled. If a directory is
* found, a relative path to the data directory is returned.
*/
-#ifdef WIN32
- if (!pgwin32_is_junction(sourcepath))
- PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
-#else
if (lstat(sourcepath, &st) < 0)
{
ereport(ERROR,
@@ -330,7 +324,6 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
if (!S_ISLNK(st.st_mode))
PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
-#endif
/*
* In presence of a link or a junction point, return the path pointing to.
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index dc20122c89..324ccf7783 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -384,11 +384,7 @@ scan_directory(const char *basedir, const char *subdir, bool sizeonly)
if (!sizeonly)
scan_file(fn, segmentno);
}
-#ifndef WIN32
else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
-#else
- else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
-#endif
{
/*
* If going through the entries of pg_tblspc, we assume to operate
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 6cb288f099..7d98283c79 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -431,11 +431,7 @@ recurse_dir(const char *datadir, const char *parentpath,
/* recurse to handle subdirectories */
recurse_dir(datadir, path, callback);
}
-#ifndef WIN32
else if (S_ISLNK(fst.st_mode))
-#else
- else if (pgwin32_is_junction(fullpath))
-#endif
{
#if defined(HAVE_READLINK) || defined(WIN32)
char link_target[MAXPGPATH];
diff --git a/src/common/file_utils.c b/src/common/file_utils.c
index 966b987d64..71c232d983 100644
--- a/src/common/file_utils.c
+++ b/src/common/file_utils.c
@@ -79,7 +79,6 @@ fsync_pgdata(const char *pg_data,
*/
xlog_is_symlink = false;
-#ifndef WIN32
{
struct stat st;
@@ -88,10 +87,6 @@ fsync_pgdata(const char *pg_data,
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
-#else
- if (pgwin32_is_junction(pg_wal))
- xlog_is_symlink = true;
-#endif
/*
* If possible, hint to the kernel that we're soon going to fsync the data
@@ -465,21 +460,5 @@ get_dirent_type(const char *path,
#endif
}
-#if defined(WIN32) && !defined(_MSC_VER)
-
- /*
- * If we're on native Windows (not Cygwin, which has its own POSIX
- * symlinks), but not using the MSVC compiler, then we're using a
- * readdir() emulation provided by the MinGW runtime that has no d_type.
- * Since the lstat() fallback code reports junction points as directories,
- * we need an extra system call to check if we should report them as
- * symlinks instead, following our convention.
- */
- if (result == PGFILETYPE_DIR &&
- !look_through_symlinks &&
- pgwin32_is_junction(path))
- result = PGFILETYPE_LNK;
-#endif
-
return result;
}
diff --git a/src/include/port.h b/src/include/port.h
index d39b04141f..4b978f61bd 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -284,7 +284,6 @@ extern int pgunlink(const char *path);
#if defined(WIN32) && !defined(__CYGWIN__)
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index b8cf2e1480..72a0dd572e 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -238,7 +238,6 @@ int setitimer(int which, const struct itimerval *value, struct itimerval *oval
*/
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index ea191e99c6..2818bfd2e9 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -362,20 +362,4 @@ pgreadlink(const char *path, char *buf, size_t size)
return r;
}
-/*
- * Assumes the file exists, so will return false if it doesn't
- * (since a nonexistent file is not a junction)
- */
-bool
-pgwin32_is_junction(const char *path)
-{
- DWORD attr = GetFileAttributes(path);
-
- if (attr == INVALID_FILE_ATTRIBUTES)
- {
- _dosmaperr(GetLastError());
- return false;
- }
- return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT);
-}
#endif /* defined(WIN32) && !defined(__CYGWIN__) */
--
2.37.1
On 2022-08-01 Mo 01:09, Thomas Munro wrote:
On Thu, Jul 28, 2022 at 9:31 PM Thomas Munro <thomas.munro@gmail.com> wrote:
There's one curious change in the draft patch attached: you can't
unlink() a junction point, you have to rmdir() it. Previously, things
that traverse directories without ever calling pgwin32_is_junction()
would see junction points as S_ISDIR() and call rmdir(), which was OK,
but now they see S_ISLNK() and call unlink(). So I taught unlink() to
try both things. Which is kinda weird, and not beautiful, especially
when combined with the existing looping weirdness.Here's a new attempt at unlink(), this time in its own patch. This
version is a little more careful about calling rmdir() only after
checking that it is a junction point, so that unlink("a directory")
fails just like on Unix (well, POSIX says that that should fail with
EPERM, not EACCES, and implementations are allowed to make it work
anyway, but it doesn't seem helpful to allow it to work there when
every OS I know of fails with EPERM or EISDIR). That check is racy,
but should be good enough for our purposes, no (see comment for a note
on that)?Longer term, I wonder if we should get rid of our use of symlinks, and
instead just put paths in a file and do our own path translation. But
for now, this patch set completes the set of junction point-based
emulations, and, IMHO, cleans up a confusing aspect of our code.As before, 0001 is just for cfbot to add an MSYS checkmark.
I'll try it out on fairywren/drongo.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
On 2022-08-01 Mo 16:06, Andrew Dunstan wrote:
On 2022-08-01 Mo 01:09, Thomas Munro wrote:
On Thu, Jul 28, 2022 at 9:31 PM Thomas Munro <thomas.munro@gmail.com> wrote:
There's one curious change in the draft patch attached: you can't
unlink() a junction point, you have to rmdir() it. Previously, things
that traverse directories without ever calling pgwin32_is_junction()
would see junction points as S_ISDIR() and call rmdir(), which was OK,
but now they see S_ISLNK() and call unlink(). So I taught unlink() to
try both things. Which is kinda weird, and not beautiful, especially
when combined with the existing looping weirdness.Here's a new attempt at unlink(), this time in its own patch. This
version is a little more careful about calling rmdir() only after
checking that it is a junction point, so that unlink("a directory")
fails just like on Unix (well, POSIX says that that should fail with
EPERM, not EACCES, and implementations are allowed to make it work
anyway, but it doesn't seem helpful to allow it to work there when
every OS I know of fails with EPERM or EISDIR). That check is racy,
but should be good enough for our purposes, no (see comment for a note
on that)?Longer term, I wonder if we should get rid of our use of symlinks, and
instead just put paths in a file and do our own path translation. But
for now, this patch set completes the set of junction point-based
emulations, and, IMHO, cleans up a confusing aspect of our code.As before, 0001 is just for cfbot to add an MSYS checkmark.
I'll try it out on fairywren/drongo.
They are happy with patches 2, 3, and 4.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
On Thu, Aug 4, 2022 at 9:28 AM Andrew Dunstan <andrew@dunslane.net> wrote:
On 2022-08-01 Mo 16:06, Andrew Dunstan wrote:
I'll try it out on fairywren/drongo.
They are happy with patches 2, 3, and 4.
Thanks for testing!
If there are no objections, I'll go ahead and commit these later today.
On Thu, Aug 4, 2022 at 9:42 AM Thomas Munro <thomas.munro@gmail.com> wrote:
On Thu, Aug 4, 2022 at 9:28 AM Andrew Dunstan <andrew@dunslane.net> wrote:
On 2022-08-01 Mo 16:06, Andrew Dunstan wrote:
I'll try it out on fairywren/drongo.
They are happy with patches 2, 3, and 4.
Thanks for testing!
If there are no objections, I'll go ahead and commit these later today.
Hmm, POSIX says st_link should contain the length of a symlink's
target path, so I suppose we should probably set that even though we
never consult it. Here's a version that does that. I also removed
the rest of the now redundant #ifdef S_ISLNK conditions.
Attachments:
v4-0001-Added-Windows-with-MinGW-environment-in-Cirrus-CI.patchtext/x-patch; charset=US-ASCII; name=v4-0001-Added-Windows-with-MinGW-environment-in-Cirrus-CI.patchDownload
From 6e57c91b7b780ea51fa45e63cb201ff4240a416b Mon Sep 17 00:00:00 2001
From: Melih Mutlu <memutlu@microsoft.com>
Date: Mon, 21 Feb 2022 14:46:05 +0300
Subject: [PATCH v4 1/4] Added Windows with MinGW environment in Cirrus CI
---
.cirrus.yml | 79 +++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 65 insertions(+), 14 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index 81eb8a9996..9820915f35 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -23,7 +23,6 @@ env:
CHECKFLAGS: -Otarget
PROVE_FLAGS: --timer
PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
- TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
PG_TEST_EXTRA: kerberos ldap ssl
@@ -338,13 +337,30 @@ task:
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
+WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
+ env:
+ # Half the allowed per-user CPU cores
+ CPUS: 4
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
+
+ # Avoids port conflicts between concurrent tap test runs
+ PG_TEST_USE_UNIX_SOCKETS: 1
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+
+ sysinfo_script: |
+ chcp
+ systeminfo
+ powershell -Command get-psdrive -psprovider filesystem
+ set
+
task:
+ << : *WINDOWS_ENVIRONMENT_BASE
name: Windows - Server 2019, VS 2019
env:
- # Half the allowed per-user CPU cores
- CPUS: 4
-
# Our windows infrastructure doesn't have test concurrency above the level
# of a single vcregress test target. Due to that, it's useful to run prove
# with multiple jobs. For the other tasks it isn't, because two sources
@@ -354,15 +370,11 @@ task:
# likely can be improved upon further.
PROVE_FLAGS: -j10 --timer
- # The default cirrus working dir is in a directory msbuild complains about
- CIRRUS_WORKING_DIR: "c:/cirrus"
# Avoid re-installing over and over
NO_TEMP_INSTALL: 1
# git's tar doesn't deal with drive letters, see
# https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
TAR: "c:/windows/system32/tar.exe"
- # Avoids port conflicts between concurrent tap test runs
- PG_TEST_USE_UNIX_SOCKETS: 1
PG_REGRESS_SOCK_DIR: "c:/cirrus/"
# -m enables parallelism
# verbosity:minimal + Summary reduce verbosity, while keeping a summary of
@@ -393,12 +405,6 @@ task:
cpu: $CPUS
memory: 4G
- sysinfo_script: |
- chcp
- systeminfo
- powershell -Command get-psdrive -psprovider filesystem
- set
-
setup_additional_packages_script: |
REM choco install -y --no-progress ...
@@ -456,6 +462,51 @@ task:
path: "crashlog-*.txt"
type: text/plain
+task:
+ << : *WINDOWS_ENVIRONMENT_BASE
+ name: Windows - Server 2019, MinGW64
+ windows_container:
+ image: $CONTAINER_REPO/windows_ci_mingw64:latest
+ cpu: $CPUS
+ memory: 4G
+ env:
+ CCACHE_DIR: C:/msys64/ccache
+ BUILD_DIR: "%CIRRUS_WORKING_DIR%/build"
+
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+
+ mingw_info_script:
+ - C:\msys64\usr\bin\bash.exe -lc "where gcc"
+ - C:\msys64\usr\bin\bash.exe -lc "gcc --version"
+ - C:\msys64\usr\bin\bash.exe -lc "where perl"
+ - C:\msys64\usr\bin\bash.exe -lc "perl --version"
+
+ configure_script:
+ - C:\msys64\usr\bin\bash.exe -lc "mkdir %BUILD_DIR% &&
+ cd %BUILD_DIR% &&
+ %CIRRUS_WORKING_DIR%/configure
+ --host=x86_64-w64-mingw32
+ --enable-cassert
+ --enable-tap-tests
+ --with-icu
+ --with-libxml
+ --with-libxslt
+ --with-lz4
+ --enable-debug
+ CC='ccache gcc'
+ CXX='ccache g++'"
+
+ build_script:
+ C:\msys64\usr\bin\bash.exe -lc "cd %BUILD_DIR% && make -s world-bin -j${CPUS}"
+
+ upload_caches: ccache
+
+ tests_script:
+ - set "NoDefaultCurrentDirectoryInExePath=0"
+ - C:\msys64\usr\bin\bash.exe -lc "cd %BUILD_DIR% && make -s ${CHECK} ${CHECKFLAGS} -j${CPUS} TMPDIR=%BUILD_DIR%/tmp_install"
+
+ on_failure: *on_failure
task:
name: CompilerWarnings
--
2.37.1
v4-0002-Provide-lstat-for-Windows.patchtext/x-patch; charset=US-ASCII; name=v4-0002-Provide-lstat-for-Windows.patchDownload
From f8be96a1c712b7ffe42445108022b9978825ce97 Mon Sep 17 00:00:00 2001
From: Thomas Munro <tmunro@postgresql.org>
Date: Fri, 5 Aug 2022 16:41:34 +1200
Subject: [PATCH v4 2/4] Provide lstat() for Windows.
Junction points will be reported with S_ISLNK(x.st_mode), simulating
POSIX lstat(). stat() will follow pseudo-symlinks, like in POSIX (but
only one level before giving up, unlike in POSIX).
Tested-by: Andrew Dunstan <andrew@dunslane.net>
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/include/port/win32_port.h | 18 +++++-
src/port/win32stat.c | 104 ++++++++++++++++++++++++++++++++--
2 files changed, 117 insertions(+), 5 deletions(-)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index 0eaa97561a..c27b34de5b 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -278,10 +278,11 @@ struct stat /* This should match struct __stat64 */
extern int _pgfstat64(int fileno, struct stat *buf);
extern int _pgstat64(const char *name, struct stat *buf);
+extern int _pglstat64(const char *name, struct stat *buf);
#define fstat(fileno, sb) _pgfstat64(fileno, sb)
#define stat(path, sb) _pgstat64(path, sb)
-#define lstat(path, sb) _pgstat64(path, sb)
+#define lstat(path, sb) _pglstat64(path, sb)
/* These macros are not provided by older MinGW, nor by MSVC */
#ifndef S_IRUSR
@@ -327,6 +328,21 @@ extern int _pgstat64(const char *name, struct stat *buf);
#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
#endif
+/*
+ * In order for lstat() to be able to report junction points as symlinks, we
+ * need to hijack a bit in st_mode, since neither MSVC nor MinGW provides
+ * S_ISLNK and there aren't any spare bits. We'll steal the one for character
+ * devices, because we don't otherwise make use of those.
+ */
+#ifdef S_ISLNK
+#error "S_ISLNK is already defined"
+#endif
+#ifdef S_IFLNK
+#error "S_IFLNK is already defined"
+#endif
+#define S_IFLNK S_IFCHR
+#define S_ISLNK(m) (((m) & S_IFLNK) == S_IFLNK)
+
/*
* Supplement to <fcntl.h>.
* This is the same value as _O_NOINHERIT in the MS header file. This is
diff --git a/src/port/win32stat.c b/src/port/win32stat.c
index e03ed5f35c..b10e35886e 100644
--- a/src/port/win32stat.c
+++ b/src/port/win32stat.c
@@ -15,7 +15,11 @@
#ifdef WIN32
+#define UMDF_USING_NTSTATUS
+
#include "c.h"
+#include "port/win32ntdll.h"
+
#include <windows.h>
/*
@@ -107,12 +111,10 @@ fileinfo_to_stat(HANDLE hFile, struct stat *buf)
}
/*
- * Windows implementation of stat().
- *
- * This currently also implements lstat(), though perhaps that should change.
+ * Windows implementation of lstat().
*/
int
-_pgstat64(const char *name, struct stat *buf)
+_pglstat64(const char *name, struct stat *buf)
{
/*
* Our open wrapper will report STATUS_DELETE_PENDING as ENOENT. We
@@ -129,10 +131,104 @@ _pgstat64(const char *name, struct stat *buf)
ret = fileinfo_to_stat(hFile, buf);
+ /*
+ * Junction points appear as directories to fileinfo_to_stat(), so we'll
+ * need to do a bit more work to distinguish them.
+ */
+ if (ret == 0 && S_ISDIR(buf->st_mode))
+ {
+ char next[MAXPGPATH];
+ ssize_t size;
+
+ /*
+ * POSIX says we need to put the length of the target path into
+ * st_size. Use readlink() to get it, or learn that this is not a
+ * junction point.
+ */
+ size = readlink(name, next, sizeof(next));
+ if (size < 0)
+ {
+ if (errno == EACCES &&
+ pg_RtlGetLastNtStatus() == STATUS_DELETE_PENDING)
+ {
+ /* Unlinked underneath us. */
+ errno = ENOENT;
+ ret = -1;
+ }
+ else if (errno == EINVAL)
+ {
+ /* It's not a junction point, nothing to do. */
+ }
+ else
+ {
+ /* Some other failure. */
+ ret = -1;
+ }
+ }
+ else
+ {
+ /* It's a junction point, so report it as a symlink. */
+ buf->st_mode &= ~S_IFDIR;
+ buf->st_mode |= S_IFLNK;
+ buf->st_size = size;
+ }
+ }
+
CloseHandle(hFile);
return ret;
}
+/*
+ * Windows implementation of stat().
+ */
+int
+_pgstat64(const char *name, struct stat *buf)
+{
+ int ret;
+
+ ret = _pglstat64(name, buf);
+
+ /* Do we need to follow a symlink (junction point)? */
+ if (ret == 0 && S_ISLNK(buf->st_mode))
+ {
+ char next[MAXPGPATH];
+ ssize_t size;
+
+ size = readlink(name, next, sizeof(next));
+ if (size < 0)
+ {
+ if (errno == EACCES &&
+ pg_RtlGetLastNtStatus() == STATUS_DELETE_PENDING)
+ {
+ /* Unlinked underneath us. */
+ errno = ENOENT;
+ }
+ return -1;
+ }
+ if (size >= sizeof(next))
+ {
+ errno = ENAMETOOLONG;
+ return -1;
+ }
+ next[size] = 0;
+
+ ret = _pglstat64(next, buf);
+ if (ret == 0 && S_ISLNK(buf->st_mode))
+ {
+ /*
+ * We're only prepared to go one hop, because we only expect to
+ * deal with the simple cases that we create. The error for too
+ * many symlinks is supposed to be ELOOP, but Windows hasn't got
+ * it.
+ */
+ errno = EIO;
+ return -1;
+ }
+ }
+
+ return ret;
+}
+
/*
* Windows implementation of fstat().
*/
--
2.37.1
v4-0004-Replace-pgwin32_is_junction-with-lstat.patchtext/x-patch; charset=US-ASCII; name=v4-0004-Replace-pgwin32_is_junction-with-lstat.patchDownload
From 9fa720ed83ea394a5c701a9ec4aee3336d23b6b0 Mon Sep 17 00:00:00 2001
From: Thomas Munro <tmunro@postgresql.org>
Date: Fri, 5 Aug 2022 09:52:26 +1200
Subject: [PATCH v4 4/4] Replace pgwin32_is_junction() with lstat().
Now that lstat() reports junction points with S_IFLNK and unlink() can
unlink them, there is no need for conditional code for Windows in a few
places. Sometimes that was expressed by testing for WIN32, other times
by testing for S_ISLNK.
The coding around pgwin32_is_junction() was a bit suspect anyway, as we
never checked for errors, and we also know that errors can be spuriously
reported because of transient sharing violations on this OS. The
lstat()-based code has handling for that.
This also reverts 4fc6b6ee on master only. That was done only because
lstat() didn't previously work for symlinks (junction points), but isn't
needed anymore.
Tested-by: Andrew Dunstan <andrew@dunslane.net>
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/backend/commands/tablespace.c | 7 +------
src/backend/replication/basebackup.c | 12 +-----------
src/backend/storage/file/fd.c | 5 -----
src/backend/utils/adt/misc.c | 7 -------
src/bin/pg_checksums/pg_checksums.c | 4 ----
src/bin/pg_rewind/file_ops.c | 4 ----
src/common/file_utils.c | 23 -----------------------
src/include/port.h | 1 -
src/include/port/win32_port.h | 1 -
src/port/dirmod.c | 16 ----------------
10 files changed, 2 insertions(+), 78 deletions(-)
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 526e82e388..f260b484fc 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -792,8 +792,7 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo)
/*
* Try to remove the symlink. We must however deal with the possibility
* that it's a directory instead of a symlink --- this could happen during
- * WAL replay (see TablespaceCreateDbspace), and it is also the case on
- * Windows where junction points lstat() as directories.
+ * WAL replay (see TablespaceCreateDbspace).
*
* Note: in the redo case, we'll return true if this final step fails;
* there's no point in retrying it. Also, ENOENT should provoke no more
@@ -823,7 +822,6 @@ remove_symlink:
linkloc)));
}
}
-#ifdef S_ISLNK
else if (S_ISLNK(st.st_mode))
{
if (unlink(linkloc) < 0)
@@ -836,7 +834,6 @@ remove_symlink:
linkloc)));
}
}
-#endif
else
{
/* Refuse to remove anything that's not a directory or symlink */
@@ -914,7 +911,6 @@ remove_tablespace_symlink(const char *linkloc)
errmsg("could not remove directory \"%s\": %m",
linkloc)));
}
-#ifdef S_ISLNK
else if (S_ISLNK(st.st_mode))
{
if (unlink(linkloc) < 0 && errno != ENOENT)
@@ -923,7 +919,6 @@ remove_tablespace_symlink(const char *linkloc)
errmsg("could not remove symbolic link \"%s\": %m",
linkloc)));
}
-#endif
else
{
/* Refuse to remove anything that's not a directory or symlink */
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 5e457f9be9..deeddd09a9 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -1322,13 +1322,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
}
/* Allow symbolic links in pg_tblspc only */
- if (strcmp(path, "./pg_tblspc") == 0 &&
-#ifndef WIN32
- S_ISLNK(statbuf.st_mode)
-#else
- pgwin32_is_junction(pathbuf)
-#endif
- )
+ if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
{
char linkpath[MAXPGPATH];
int rllen;
@@ -1798,11 +1792,7 @@ static void
convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
{
/* If symlink, write it as a directory anyway */
-#ifndef WIN32
if (S_ISLNK(statbuf->st_mode))
-#else
- if (pgwin32_is_junction(pathbuf))
-#endif
statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
}
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index ccb540d617..efb34d4dcb 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -3365,7 +3365,6 @@ SyncDataDirectory(void)
*/
xlog_is_symlink = false;
-#ifndef WIN32
{
struct stat st;
@@ -3377,10 +3376,6 @@ SyncDataDirectory(void)
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
-#else
- if (pgwin32_is_junction("pg_wal"))
- xlog_is_symlink = true;
-#endif
#ifdef HAVE_SYNCFS
if (recovery_init_sync_method == RECOVERY_INIT_SYNC_METHOD_SYNCFS)
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index af0d924459..d35b5d1f4f 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -283,9 +283,7 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
char sourcepath[MAXPGPATH];
char targetpath[MAXPGPATH];
int rllen;
-#ifndef WIN32
struct stat st;
-#endif
/*
* It's useful to apply this function to pg_class.reltablespace, wherein
@@ -314,10 +312,6 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
* created with allow_in_place_tablespaces enabled. If a directory is
* found, a relative path to the data directory is returned.
*/
-#ifdef WIN32
- if (!pgwin32_is_junction(sourcepath))
- PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
-#else
if (lstat(sourcepath, &st) < 0)
{
ereport(ERROR,
@@ -328,7 +322,6 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
if (!S_ISLNK(st.st_mode))
PG_RETURN_TEXT_P(cstring_to_text(sourcepath));
-#endif
/*
* In presence of a link or a junction point, return the path pointing to.
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index dc20122c89..324ccf7783 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -384,11 +384,7 @@ scan_directory(const char *basedir, const char *subdir, bool sizeonly)
if (!sizeonly)
scan_file(fn, segmentno);
}
-#ifndef WIN32
else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
-#else
- else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
-#endif
{
/*
* If going through the entries of pg_tblspc, we assume to operate
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 5e6d8b89c4..db190bcba7 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -431,11 +431,7 @@ recurse_dir(const char *datadir, const char *parentpath,
/* recurse to handle subdirectories */
recurse_dir(datadir, path, callback);
}
-#ifndef WIN32
else if (S_ISLNK(fst.st_mode))
-#else
- else if (pgwin32_is_junction(fullpath))
-#endif
{
char link_target[MAXPGPATH];
int len;
diff --git a/src/common/file_utils.c b/src/common/file_utils.c
index 966b987d64..df4d6d240c 100644
--- a/src/common/file_utils.c
+++ b/src/common/file_utils.c
@@ -79,7 +79,6 @@ fsync_pgdata(const char *pg_data,
*/
xlog_is_symlink = false;
-#ifndef WIN32
{
struct stat st;
@@ -88,10 +87,6 @@ fsync_pgdata(const char *pg_data,
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
-#else
- if (pgwin32_is_junction(pg_wal))
- xlog_is_symlink = true;
-#endif
/*
* If possible, hint to the kernel that we're soon going to fsync the data
@@ -459,27 +454,9 @@ get_dirent_type(const char *path,
result = PGFILETYPE_REG;
else if (S_ISDIR(fst.st_mode))
result = PGFILETYPE_DIR;
-#ifdef S_ISLNK
else if (S_ISLNK(fst.st_mode))
result = PGFILETYPE_LNK;
-#endif
}
-#if defined(WIN32) && !defined(_MSC_VER)
-
- /*
- * If we're on native Windows (not Cygwin, which has its own POSIX
- * symlinks), but not using the MSVC compiler, then we're using a
- * readdir() emulation provided by the MinGW runtime that has no d_type.
- * Since the lstat() fallback code reports junction points as directories,
- * we need an extra system call to check if we should report them as
- * symlinks instead, following our convention.
- */
- if (result == PGFILETYPE_DIR &&
- !look_through_symlinks &&
- pgwin32_is_junction(path))
- result = PGFILETYPE_LNK;
-#endif
-
return result;
}
diff --git a/src/include/port.h b/src/include/port.h
index 14b640fe33..feb2ae840d 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -284,7 +284,6 @@ extern int pgunlink(const char *path);
#if defined(WIN32) && !defined(__CYGWIN__)
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h
index c27b34de5b..a65996cce5 100644
--- a/src/include/port/win32_port.h
+++ b/src/include/port/win32_port.h
@@ -230,7 +230,6 @@ int setitimer(int which, const struct itimerval *value, struct itimerval *oval
*/
extern int pgsymlink(const char *oldpath, const char *newpath);
extern int pgreadlink(const char *path, char *buf, size_t size);
-extern bool pgwin32_is_junction(const char *path);
#define symlink(oldpath, newpath) pgsymlink(oldpath, newpath)
#define readlink(path, buf, size) pgreadlink(path, buf, size)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index ea191e99c6..2818bfd2e9 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -362,20 +362,4 @@ pgreadlink(const char *path, char *buf, size_t size)
return r;
}
-/*
- * Assumes the file exists, so will return false if it doesn't
- * (since a nonexistent file is not a junction)
- */
-bool
-pgwin32_is_junction(const char *path)
-{
- DWORD attr = GetFileAttributes(path);
-
- if (attr == INVALID_FILE_ATTRIBUTES)
- {
- _dosmaperr(GetLastError());
- return false;
- }
- return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT);
-}
#endif /* defined(WIN32) && !defined(__CYGWIN__) */
--
2.37.1
v4-0003-Make-unlink-work-for-junction-points-on-Windows.patchtext/x-patch; charset=US-ASCII; name=v4-0003-Make-unlink-work-for-junction-points-on-Windows.patchDownload
From af3e1f2e5fd505299d9062a8aedc6765983aebc2 Mon Sep 17 00:00:00 2001
From: Thomas Munro <tmunro@postgresql.org>
Date: Fri, 5 Aug 2022 16:41:50 +1200
Subject: [PATCH v4 3/4] Make unlink() work for junction points on Windows.
To support harmonization of Windows and Unix code, teach our unlink()
wrapper that junction points need to be unlinked with rmdir() on
Windows.
Tested-by: Andrew Dunstan <andrew@dunslane.net>
Discussion: https://postgr.es/m/CA%2BhUKGLfOOeyZpm5ByVcAt7x5Pn-%3DxGRNCvgiUPVVzjFLtnY0w%40mail.gmail.com
---
src/port/dirmod.c | 28 +++++++++++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 7ce042e75d..ea191e99c6 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -99,6 +99,32 @@ int
pgunlink(const char *path)
{
int loops = 0;
+ struct stat st;
+
+ /*
+ * This function might be called for a regular file or for a junction
+ * point (which we use to emulate symlinks). The latter must be unlinked
+ * with rmdir() on Windows. Before we worry about any of that, let's see
+ * if we can unlink directly, since that's expected to be the most common
+ * case.
+ */
+ if (unlink(path) == 0)
+ return 0;
+ if (errno != EACCES)
+ return -1;
+
+ /*
+ * EACCES is reported for many reasons including unlink() of a junction
+ * point. Check if that's the case so we can redirect to rmdir().
+ *
+ * Note that by checking only once, we can't cope with a path that changes
+ * from regular file to junction point underneath us while we're retrying
+ * due to sharing violations, but that seems unlikely. We could perhaps
+ * prevent that by holding a file handle ourselves across the lstat() and
+ * the retry loop, but that seems like over-engineering for now.
+ */
+ if (lstat(path, &st) < 0)
+ return -1;
/*
* We need to loop because even though PostgreSQL uses flags that allow
@@ -107,7 +133,7 @@ pgunlink(const char *path)
* someone else to close the file, as the caller might be holding locks
* and blocking other backends.
*/
- while (unlink(path))
+ while ((S_ISLNK(st.st_mode) ? rmdir(path) : unlink(path)) < 0)
{
if (errno != EACCES)
return -1;
--
2.37.1
On Fri, Aug 5, 2022 at 9:17 PM Thomas Munro <thomas.munro@gmail.com> wrote:
Hmm, POSIX says st_link should contain the length of a symlink's
target path, so I suppose we should probably set that even though we
never consult it. Here's a version that does that. I also removed
the rest of the now redundant #ifdef S_ISLNK conditions.
Pushed.
Hmm, this stuff could *really* use a little test framework that's run
by check-world, that exercises these various replacement operations.
But I also suspect that problems in this area are likely to be due to
concurrency. It's hard to make a simple test that simulates the case
where a file is unlinked between system calls within stat() and hits
the STATUS_DELETE_PENDING case. That check is code I cargo-culted in
this patch. So much of the stuff we've had in the tree relating to
that area has been wrong in the past...
On 2022-08-06 08:02, Thomas Munro wrote:
Pushed.
Hmm, this stuff could *really* use a little test framework that's run
by check-world, that exercises these various replacement operations.
But I also suspect that problems in this area are likely to be due to
concurrency. It's hard to make a simple test that simulates the case
where a file is unlinked between system calls within stat() and hits
the STATUS_DELETE_PENDING case. That check is code I cargo-culted in
this patch. So much of the stuff we've had in the tree relating to
that area has been wrong in the past...
Hello, hackers!
initdb on my windows 10 system stopped working after the commit
c5cb8f3b: "Provide lstat() for Windows."
The error message is: creating directory C:/HOME/data ... initdb:
error: could not create directory "C:/HOME": File exists
"C:/HOME" is the junction point to the second volume on my hard drive -
"\??\Volume{GUID}\" which name pgreadlink() erroneously strips here:
https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/dirmod.c#L357.
So initdb could not stat the file with name "Volume{GUID}", tried to
create it and failed.
With the attached patch initdb works fine again.
--
regards,
Roman
Attachments:
dirmod.c.patchtext/x-diff; name=dirmod.c.patchDownload
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 2818bfd2e95..9d73620a6ba 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -354,7 +354,8 @@ pgreadlink(const char *path, char *buf, size_t size)
* If the path starts with "\??\", which it will do in most (all?) cases,
* strip those out.
*/
- if (r > 4 && strncmp(buf, "\\??\\", 4) == 0)
+ if (r > 4 && strncmp(buf, "\\??\\", 4) == 0 &&
+ strncmp(buf, "\\??\\Volume", 10) != 0)
{
memmove(buf, buf + 4, strlen(buf + 4) + 1);
r -= 4;
On Mon, Aug 8, 2022 at 8:23 PM <r.zharkov@postgrespro.ru> wrote:
initdb on my windows 10 system stopped working after the commit
c5cb8f3b: "Provide lstat() for Windows."
The error message is: creating directory C:/HOME/data ... initdb:
error: could not create directory "C:/HOME": File exists"C:/HOME" is the junction point to the second volume on my hard drive -
"\??\Volume{GUID}\" which name pgreadlink() erroneously strips here:
https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/dirmod.c#L357.
So initdb could not stat the file with name "Volume{GUID}", tried to
create it and failed.
With the attached patch initdb works fine again.
- if (r > 4 && strncmp(buf, "\\??\\", 4) == 0)
+ if (r > 4 && strncmp(buf, "\\??\\", 4) == 0 &&
+ strncmp(buf, "\\??\\Volume", 10) != 0)
{
memmove(buf, buf + 4, strlen(buf + 4) + 1);
r -= 4;
Hmm. I suppose the problem must be in pg_mkdir_p(). Our symlink()
emulation usually adds this "\??\" prefix (making it an "NT object
path"?), because junction points only work if they are in that format.
Then our readlink() emulation removes it again, but in the case of
your \??\Volume{GUID} path, created by you, not our symlink()
emulation, removing "\??\" apparently makes it unopenable with
CreateFile() (I guess that's what fails? What's the error?). So your
patch just says: don't strip "\??\" if it's followed by "Volume".
I don't understand all the kinds of DOS, Windows and NT paths (let me
take a moment to say how much I love Unix), but here's a guess: could
it be that NT "\??\C:\foo" = DOS "C:\foo", but NT "\??\Volume..." =
DOS "\Volume..."? In other words, if it hasn't got a drive letter,
maybe it still needs an initial "\" (or if not that, then *something*
special, because otherwise it looks like a relative path). Would it
be better to say: if it doesn't begin with "\??\X:", where X could be
any letter, then don't modify it?
Maybe [1]https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html has some clues. It seems to give the info in a higher
density form than the Windows docs (at least to the uninitiated like
me wanting a quick overview with examples). Hmm, I wonder if we could
get away from doing our own path mangling and use some of the proper
library calls mentioned on that page...
[1]: https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html
On Tue, Aug 9, 2022 at 8:30 AM Thomas Munro <thomas.munro@gmail.com> wrote:
On Mon, Aug 8, 2022 at 8:23 PM <r.zharkov@postgrespro.ru> wrote:
"C:/HOME" is the junction point to the second volume on my hard drive -
"\??\Volume{GUID}\" which name pgreadlink() erroneously strips here:
https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/dirmod.c#L357.
... Would it
be better to say: if it doesn't begin with "\??\X:", where X could be
any letter, then don't modify it?
Concretely, I wonder if this is a good fix at least in the short term.
Does this work for you, and do the logic and explanation make sense?
Attachments:
0001-Fix-readlink-for-more-complex-Windows-paths.patchtext/x-patch; charset=US-ASCII; name=0001-Fix-readlink-for-more-complex-Windows-paths.patchDownload
From 9772b2bf5a1b11605ea16cfd17b3b50e0274aadf Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Tue, 9 Aug 2022 10:29:08 +1200
Subject: [PATCH] Fix readlink() for more complex Windows paths.
Our symlink() and readlink() replacements perform a very naive
transformation from "DOS" paths to "NT" paths and back, as required by
the junction point APIs. This was enough for the simple paths we expect
users to provide for tablespaces, for example 'd:\my\fast\storage'.
Since commit c5cb8f3b taught stat() to follow symlinks, and since initdb
uses pg_mkdir_p(), and that examines parent directories, our humble
readlink() implementation can now be exposed to junction points not of
PostgreSQL origin. Those might be corrupted by our naive path mangling,
which doesn't really understand NT paths in general.
It's possible that we should investigate using Rtl*() library routines
to do path transformation, but for now, simply decline to transform
paths that don't fit the simple drive-qualified pattern. That means
that readlink() returns an NT path, which works just as well as a DOS
path when following the link.
Reported-by: Roman Zharkov <r.zharkov@postgrespro.ru>
Discussion: https://postgr.es/m/4590c37927d7b8ee84f9855d83229018%40postgrespro.ru
---
src/port/dirmod.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 2818bfd2e9..821563c290 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -352,9 +352,11 @@ pgreadlink(const char *path, char *buf, size_t size)
/*
* If the path starts with "\??\", which it will do in most (all?) cases,
- * strip those out.
+ * strip those out, but only if it's of the simple drive-letter-qualified
+ * type that we know how to transform from NT to DOS format.
*/
- if (r > 4 && strncmp(buf, "\\??\\", 4) == 0)
+ if (r >= 6 && strncmp(buf, "\\??\\", 4) == 0 && isalpha(buf[4]) &&
+ buf[5] == ':')
{
memmove(buf, buf + 4, strlen(buf + 4) + 1);
r -= 4;
--
2.35.1
On 2022-08-09 03:30, Thomas Munro wrote:
Then our readlink() emulation removes it again, but in the case of
your \??\Volume{GUID} path, created by you, not our symlink()
emulation, removing "\??\" apparently makes it unopenable with
CreateFile() (I guess that's what fails? What's the error?). So your
patch just says: don't strip "\??\" if it's followed by "Volume".
Sorry, I thought wrong that everyone sees the backtrace on my screen.
Failes the CreateFile() function with fileName = "Volume{GUID}\" at [1]https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/open.c#L86.
And the GetLastError() returnes 2 (ERROR_FILE_NOT_FOUND).
Call Stack:
initdb.exe!pgwin32_open_handle(const char * fileName, ...) Line 111 C
initdb.exe!_pglstat64(const char * name, stat * buf) Line 128 C
initdb.exe!_pgstat64(const char * name, stat * buf) Line 221 C
initdb.exe!pg_mkdir_p(char * path, int omode) Line 123 C
initdb.exe!create_data_directory() Line 2537 C
initdb.exe!initialize_data_directory() Line 2696 C
initdb.exe!main(int argc, char * * argv) Line 3102 C
I don't understand all the kinds of DOS, Windows and NT paths (let me
take a moment to say how much I love Unix), but here's a guess: could
it be that NT "\??\C:\foo" = DOS "C:\foo", but NT "\??\Volume..." =
DOS "\Volume..."? In other words, if it hasn't got a drive letter,
maybe it still needs an initial "\" (or if not that, then *something*
special, because otherwise it looks like a relative path).
It seems to me, when we call CreateFile() Windows Object Manager
searches
DOS devices (drive letters in our case) in DOS Device namespaces.
But it doesn't search the "Volume{GUID}" devices which must be named as
"\\?\Volume{GUID}\" [2]https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-volume.
Would it be better to say: if it doesn't begin with "\??\X:", where X
could be any letter, then don't modify it?
I think it will be better.
[1]: https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/open.c#L86
https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/open.c#L86
[2]: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-volume
https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-volume
On 2022-08-09 05:44, Thomas Munro wrote:
On Tue, Aug 9, 2022 at 8:30 AM Thomas Munro <thomas.munro@gmail.com>
wrote:On Mon, Aug 8, 2022 at 8:23 PM <r.zharkov@postgrespro.ru> wrote:
"C:/HOME" is the junction point to the second volume on my hard drive -
"\??\Volume{GUID}\" which name pgreadlink() erroneously strips here:
https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/dirmod.c#L357.... Would it
be better to say: if it doesn't begin with "\??\X:", where X could be
any letter, then don't modify it?Concretely, I wonder if this is a good fix at least in the short term.
Does this work for you, and do the logic and explanation make sense?
Yes, this patch works well with my junction points.
I checked a few variants:
21.07.2022 15:11 <JUNCTION> HOME [\??\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test1 [\\?\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test2 [\\.\Volume{GUID}\]
09.08.2022 15:17 <JUNCTION> Test3 [\??\Volume{GUID}\]
09.08.2022 15:27 <JUNCTION> Test4 [C:\temp\1]
09.08.2022 15:28 <JUNCTION> Test5 [C:\HOME\Temp\1]
After hours of reading the documentation and debugging, it seems to me
we can use REPARSE_GUID_DATA_BUFFER structure instead of our
REPARSE_JUNCTION_DATA_BUFFER [1]https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-reparse_guid_data_buffer. DataBuffer doesn't contain any
prefixes,
so we don't need to strip them. But we still need to construct a correct
volume name if a junction point is a volume mount point. Is it worth to
check this idea?
[1]: https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-reparse_guid_data_buffer
https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-reparse_guid_data_buffer
On Tue, Aug 9, 2022 at 10:59 PM <r.zharkov@postgrespro.ru> wrote:
On 2022-08-09 05:44, Thomas Munro wrote:
On Tue, Aug 9, 2022 at 8:30 AM Thomas Munro <thomas.munro@gmail.com>
wrote:On Mon, Aug 8, 2022 at 8:23 PM <r.zharkov@postgrespro.ru> wrote:
"C:/HOME" is the junction point to the second volume on my hard drive -
"\??\Volume{GUID}\" which name pgreadlink() erroneously strips here:
https://github.com/postgres/postgres/blob/7e29a79a46d30dc236d097825ab849158929d977/src/port/dirmod.c#L357.... Would it
be better to say: if it doesn't begin with "\??\X:", where X could be
any letter, then don't modify it?Concretely, I wonder if this is a good fix at least in the short term.
Does this work for you, and do the logic and explanation make sense?Yes, this patch works well with my junction points.
Thanks for testing! I did a bit more reading on this stuff, so that I
could update the comments with the correct terminology from Windows
APIs. I also realised that the pattern we could accept to symlink()
and expect to work is not just "C:..." (could be
RtlPathTypeDriveRelative, which wouldn't actually work in a junction
point) but "C:\..." (RtlPathTypeDriveAbsolute). I tweaked it a bit to
test for that.
I checked a few variants:
21.07.2022 15:11 <JUNCTION> HOME [\??\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test1 [\\?\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test2 [\\.\Volume{GUID}\]
09.08.2022 15:17 <JUNCTION> Test3 [\??\Volume{GUID}\]
09.08.2022 15:27 <JUNCTION> Test4 [C:\temp\1]
09.08.2022 15:28 <JUNCTION> Test5 [C:\HOME\Temp\1]
One more thing I wondered about, now that we're following junctions
outside PGDATA: can a junction point to another junction? If so, I
didn't allow for that: stat() gives up after one hop, because I
figured that was enough for the stuff we expect inside PGDATA and I
couldn't find any evidence in the man pages that referred to chains.
But if you *are* allowed to create a junction "c:\huey" that points to
junction "c:\dewey" that points to "c:\louie", and then you do initdb
-D c:\huey\pgdata, then I guess it would fail. Would you mind
checking if that is a real possibility, and if so, testing this
chain-following patch to see if it fixes it?
After hours of reading the documentation and debugging, it seems to me
we can use REPARSE_GUID_DATA_BUFFER structure instead of our
REPARSE_JUNCTION_DATA_BUFFER [1]. DataBuffer doesn't contain any
prefixes,
so we don't need to strip them. But we still need to construct a correct
volume name if a junction point is a volume mount point. Is it worth to
check this idea?
I don't know. I think I prefer our current approach, because it can
handle anything (raw/full NT paths) and doesn't try to be very clever,
and I don't want to change to a different scheme for no real
benefit...
Attachments:
v2-0001-Fix-readlink-for-general-Windows-junction-points.patchtext/x-patch; charset=US-ASCII; name=v2-0001-Fix-readlink-for-general-Windows-junction-points.patchDownload
From 6ed3edc77275aa51d1d0b1012c0a36db65735d39 Mon Sep 17 00:00:00 2001
From: Thomas Munro <tmunro@postgresql.org>
Date: Thu, 11 Aug 2022 10:42:13 +1200
Subject: [PATCH v2 1/2] Fix readlink() for general Windows junction points.
Our symlink() and readlink() replacements perform a naive transformation
from DOS paths to NT paths and back, as required by the junction point
APIs. This was enough for the "drive absolute" paths we expect users to
provide for tablespaces, for example "d:\my\fast\storage".
Since commit c5cb8f3b taught stat() to follow symlinks, and since initdb
uses pg_mkdir_p(), and that examines parent directories, our humble
readlink() implementation can now be exposed to junction points not of
PostgreSQL origin. Those might be corrupted by our naive path mangling,
which doesn't really understand NT paths in general.
Simply decline to transform paths that don't look like a drive absolute
path. That means that readlink() returns the NT path directly when
checking a parent directory of PGDATA that happen to point to a drive
using "rooted" format. That works for the purposes of our stat()
emulation.
Reported-by: Roman Zharkov <r.zharkov@postgrespro.ru>
Discussion: https://postgr.es/m/4590c37927d7b8ee84f9855d83229018%40postgrespro.ru
---
src/port/dirmod.c | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/src/port/dirmod.c b/src/port/dirmod.c
index 2818bfd2e9..53310baafb 100644
--- a/src/port/dirmod.c
+++ b/src/port/dirmod.c
@@ -199,7 +199,11 @@ pgsymlink(const char *oldpath, const char *newpath)
if (dirhandle == INVALID_HANDLE_VALUE)
return -1;
- /* make sure we have an unparsed native win32 path */
+ /*
+ * We expect either an NT path or a "drive absolute" path like "C:\..."
+ * that we convert to an NT path by bolting on a prefix and converting any
+ * Unix-style path delimiters to NT-style.
+ */
if (memcmp("\\??\\", oldpath, 4) != 0)
snprintf(nativeTarget, sizeof(nativeTarget), "\\??\\%s", oldpath);
else
@@ -351,10 +355,21 @@ pgreadlink(const char *path, char *buf, size_t size)
}
/*
- * If the path starts with "\??\", which it will do in most (all?) cases,
- * strip those out.
+ * If the path starts with "\??\" followed by a "drive absolute" path
+ * (known to Windows APIs as RtlPathTypeDriveAbsolute), then strip that
+ * prefix. This undoes some of the transformation performed by
+ * pqsymlink(), to get back to a format that users are used to seeing. We
+ * don't know how to transform other path types that might be encountered
+ * outside PGDATA, so we just return them directly.
*/
- if (r > 4 && strncmp(buf, "\\??\\", 4) == 0)
+ if (r >= 7 &&
+ buf[0] == '\\' &&
+ buf[1] == '?' &&
+ buf[2] == '?' &&
+ buf[3] == '\\' &&
+ isalpha(buf[4]) &&
+ buf[5] == ':' &&
+ buf[6] == '\\')
{
memmove(buf, buf + 4, strlen(buf + 4) + 1);
r -= 4;
--
2.35.1
v2-0002-Follow-junction-point-chains-in-our-stat-emulatio.patchtext/x-patch; charset=US-ASCII; name=v2-0002-Follow-junction-point-chains-in-our-stat-emulatio.patchDownload
From 81f203af67bb0dfed1a55656496fd55211d98972 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Thu, 11 Aug 2022 12:30:48 +1200
Subject: [PATCH v2 2/2] Follow junction point chains in our stat() emulation.
Commit c5cb8f3b supposed that we'd only ever have to follow junction
points for one hop, because we don't construct longer chains of them
ourselves. But when stat()ing a parent directory supplied by the user,
we should really be able to cope with longer chains. Choose an
arbitrary cap of 8, to match the minimum acceptable value of SYMLOOP_MAX
from POSIX.
---
src/port/win32stat.c | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/src/port/win32stat.c b/src/port/win32stat.c
index 26443293d7..2ff42b9a90 100644
--- a/src/port/win32stat.c
+++ b/src/port/win32stat.c
@@ -184,16 +184,27 @@ _pglstat64(const char *name, struct stat *buf)
int
_pgstat64(const char *name, struct stat *buf)
{
+ int loops = 0;
int ret;
ret = _pglstat64(name, buf);
/* Do we need to follow a symlink (junction point)? */
- if (ret == 0 && S_ISLNK(buf->st_mode))
+ while (ret == 0 && S_ISLNK(buf->st_mode))
{
char next[MAXPGPATH];
ssize_t size;
+ if (++loops > 8)
+ {
+ /*
+ * Give up. The error for too many symlinks is supposed to be
+ * ELOOP, but Windows hasn't got it.
+ */
+ errno = EIO;
+ return -1;
+ }
+
/*
* _pglstat64() already called readlink() once to be able to fill in
* st_size, and now we need to do it again to get the path to follow.
@@ -219,17 +230,6 @@ _pgstat64(const char *name, struct stat *buf)
next[size] = 0;
ret = _pglstat64(next, buf);
- if (ret == 0 && S_ISLNK(buf->st_mode))
- {
- /*
- * We're only prepared to go one hop, because we only expect to
- * deal with the simple cases that we create. The error for too
- * many symlinks is supposed to be ELOOP, but Windows hasn't got
- * it.
- */
- errno = EIO;
- return -1;
- }
}
return ret;
--
2.35.1
On 2022-08-11 07:55, Thomas Munro wrote:
I checked a few variants:
21.07.2022 15:11 <JUNCTION> HOME [\??\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test1 [\\?\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test2 [\\.\Volume{GUID}\]
09.08.2022 15:17 <JUNCTION> Test3 [\??\Volume{GUID}\]
09.08.2022 15:27 <JUNCTION> Test4 [C:\temp\1]
09.08.2022 15:28 <JUNCTION> Test5 [C:\HOME\Temp\1]One more thing I wondered about, now that we're following junctions
outside PGDATA: can a junction point to another junction? If so, I
didn't allow for that: stat() gives up after one hop, because I
figured that was enough for the stuff we expect inside PGDATA and I
couldn't find any evidence in the man pages that referred to chains.
But if you *are* allowed to create a junction "c:\huey" that points to
junction "c:\dewey" that points to "c:\louie", and then you do initdb
-D c:\huey\pgdata, then I guess it would fail. Would you mind
checking if that is a real possibility, and if so, testing this
chain-following patch to see if it fixes it?
I made some junctions and rechecked both patches.
11.08.2022 16:11 <JUNCTION> donald [C:\huey]
11.08.2022 13:23 <JUNCTION> huey [C:\dewey]
11.08.2022 13:23 <JUNCTION> dewey [C:\louie]
11.08.2022 16:57 <DIR> louie
With the small attached patch initdb succeeded in any of these
"directories". If the junction chain is too long, initdb fails with
"could not create directory" as expected.
initdb -D huey/pgdata
...
Success.
initdb -N -D donald
...
Success.
11.08.2022 17:32 <DIR> 1
11.08.2022 17:32 <JUNCTION> 2 [C:\1]
11.08.2022 17:32 <JUNCTION> 3 [C:\2]
11.08.2022 17:32 <JUNCTION> 4 [C:\3]
11.08.2022 17:32 <JUNCTION> 5 [C:\4]
11.08.2022 17:32 <JUNCTION> 6 [C:\5]
11.08.2022 17:32 <JUNCTION> 7 [C:\6]
11.08.2022 17:32 <JUNCTION> 8 [C:\7]
11.08.2022 17:32 <JUNCTION> 9 [C:\8]
11.08.2022 17:32 <JUNCTION> 10 [C:\9]
initdb -D 10/pgdata
...
creating directory 10/pgdata ... initdb: error: could not create
directory "10": File exists
initdb -D 9/pgdata
...
Success.
Attachments:
win32stat.difftext/x-diff; name=win32stat.diffDownload
--- win32stat.c.bak 2022-08-11 17:15:10.775412600 +0700
+++ win32stat.c 2022-08-11 16:47:41.992805700 +0700
@@ -186,9 +186,12 @@
{
int loops = 0;
int ret;
+ char curr[MAXPGPATH];
ret = _pglstat64(name, buf);
+ strlcpy(curr, name, MAXPGPATH);
+
/* Do we need to follow a symlink (junction point)? */
while (ret == 0 && S_ISLNK(buf->st_mode))
{
@@ -211,7 +214,7 @@
* That could be optimized, but stat() on symlinks is probably rare
* and this way is simple.
*/
- size = readlink(name, next, sizeof(next));
+ size = readlink(curr, next, sizeof(next));
if (size < 0)
{
if (errno == EACCES &&
@@ -230,6 +233,7 @@
next[size] = 0;
ret = _pglstat64(next, buf);
+ strcpy(curr, next);
}
return ret;
On Thu, Aug 11, 2022 at 10:40 PM <r.zharkov@postgrespro.ru> wrote:
On 2022-08-11 07:55, Thomas Munro wrote:
I checked a few variants:
21.07.2022 15:11 <JUNCTION> HOME [\??\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test1 [\\?\Volume{GUID}\]
09.08.2022 15:06 <JUNCTION> Test2 [\\.\Volume{GUID}\]
09.08.2022 15:17 <JUNCTION> Test3 [\??\Volume{GUID}\]
09.08.2022 15:27 <JUNCTION> Test4 [C:\temp\1]
09.08.2022 15:28 <JUNCTION> Test5 [C:\HOME\Temp\1]One more thing I wondered about, now that we're following junctions
outside PGDATA: can a junction point to another junction? If so, I
didn't allow for that: stat() gives up after one hop, because I
figured that was enough for the stuff we expect inside PGDATA and I
couldn't find any evidence in the man pages that referred to chains.
But if you *are* allowed to create a junction "c:\huey" that points to
junction "c:\dewey" that points to "c:\louie", and then you do initdb
-D c:\huey\pgdata, then I guess it would fail. Would you mind
checking if that is a real possibility, and if so, testing this
chain-following patch to see if it fixes it?I made some junctions and rechecked both patches.
11.08.2022 16:11 <JUNCTION> donald [C:\huey]
11.08.2022 13:23 <JUNCTION> huey [C:\dewey]
11.08.2022 13:23 <JUNCTION> dewey [C:\louie]
11.08.2022 16:57 <DIR> louieWith the small attached patch initdb succeeded in any of these
"directories". If the junction chain is too long, initdb fails with
"could not create directory" as expected.
Thanks for testing and for that fix! I do intend to push this, and a
nearby fix for unlink(), but first I want to have test coverage for
all this stuff so we can demonstrate comprehensively that it works via
automated testing, otherwise it's just impossible to maintain (at
least for me, Unix guy). I have a prototype test suite based on
writing TAP tests in C and I've already found more subtle ancient bugs
around the Windows porting layer... more soon.