From 4959e4dc6d96d7634ec5ac0c8f55f11ec37b6adb Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sun, 5 Sep 2021 23:49:23 +1200 Subject: [PATCH] Use NtCreateFile() to open files on Windows. 1. Update our open() wrapper to bypass Win32 and use NT interfaces to open files. This allows us to check for NT's STATUS_DELETE_PENDING and translate it to appropriate errors. 2. Remove non-working code from our stat() wrapper that tried to handle the same problem, and reuse the open() wrapper. XXX TODO: resolve order-of-operations problem: pgwin32_open_handle() is caled by _pgstat64() before the Windows signal emulation is started up, but it might call pg_usleep(). XXX TODO: there are some macro redefinition warnings from MSVC on CI; apparently we need some well placed #define and #undef of WIN32_NO_STATUS. Discussion: https://postgr.es/m/CA%2BhUKGJz_pZTF9mckn6XgSv69%2BjGwdgLkxZ6b3NWGLBCVjqUZA%40mail.gmail.com --- configure | 6 + configure.ac | 1 + src/include/port.h | 1 + src/include/port/win32ntdll.h | 42 ++++++ src/port/open.c | 255 ++++++++++++++++++++-------------- src/port/win32ntdll.c | 68 +++++++++ src/port/win32stat.c | 164 +--------------------- src/tools/msvc/Mkvcbuild.pm | 3 +- 8 files changed, 276 insertions(+), 264 deletions(-) create mode 100644 src/include/port/win32ntdll.h create mode 100644 src/port/win32ntdll.c diff --git a/configure b/configure index c550cacd5a..adfe03f3f2 100755 --- a/configure +++ b/configure @@ -16818,6 +16818,12 @@ esac ;; esac + case " $LIBOBJS " in + *" win32ntdll.$ac_objext "* ) ;; + *) LIBOBJS="$LIBOBJS win32ntdll.$ac_objext" + ;; +esac + case " $LIBOBJS " in *" win32security.$ac_objext "* ) ;; *) LIBOBJS="$LIBOBJS win32security.$ac_objext" diff --git a/configure.ac b/configure.ac index 2ee710102f..2819b91a8c 100644 --- a/configure.ac +++ b/configure.ac @@ -1922,6 +1922,7 @@ if test "$PORTNAME" = "win32"; then AC_LIBOBJ(system) AC_LIBOBJ(win32env) AC_LIBOBJ(win32error) + AC_LIBOBJ(win32ntdll) AC_LIBOBJ(win32security) AC_LIBOBJ(win32setlocale) AC_LIBOBJ(win32stat) diff --git a/src/include/port.h b/src/include/port.h index 82f63de325..ec64be429c 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -290,6 +290,7 @@ extern bool rmtree(const char *path, bool rmtopdir); * passing of other special options. */ #define O_DIRECT 0x80000000 +extern HANDLE pgwin32_open_handle(const char *, int, bool); extern int pgwin32_open(const char *, int,...); extern FILE *pgwin32_fopen(const char *, const char *); #define open(a,b,c) pgwin32_open(a,b,c) diff --git a/src/include/port/win32ntdll.h b/src/include/port/win32ntdll.h new file mode 100644 index 0000000000..14e70c6f42 --- /dev/null +++ b/src/include/port/win32ntdll.h @@ -0,0 +1,42 @@ +/*------------------------------------------------------------------------- + * + * win32_ntdll.h + * Dynamically loaded Windows NT functions. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/win32ntdll.h + * + *------------------------------------------------------------------------- + */ + +#include "c.h" + +#include +#include + +typedef NTSTATUS (__stdcall *NtCreateFile_t)(PHANDLE FileHandle, + ACCESS_MASK DesiredAccess, + POBJECT_ATTRIBUTES ObjectAttributes, + PIO_STATUS_BLOCK IoStatusBlock, + PLARGE_INTEGER AllocationSize, + ULONG FileAttributes, + ULONG ShareAccess, + ULONG CreateDisposition, + ULONG CreateOptions, + PVOID EaBuffer, + ULONG EaLength); + +typedef BOOLEAN (__stdcall *RtlDosPathNameToNtPathName_U_t)(PCWSTR DosName, + PUNICODE_STRING NtName, + PCWSTR *PartName, + void *RelativeName); + +typedef ULONG (__stdcall *RtlNtStatusToDosError_t)(NTSTATUS Status); + +extern NtCreateFile_t pg_NtCreateFile; +extern RtlDosPathNameToNtPathName_U_t pg_RtlDosPathNameToNtPathName_U; +extern RtlNtStatusToDosError_t pg_RtlNtStatusToDosError; + +extern int initialize_ntdll(void); diff --git a/src/port/open.c b/src/port/open.c index 14c6debba9..122e1b4ab7 100644 --- a/src/port/open.c +++ b/src/port/open.c @@ -19,52 +19,38 @@ #include "postgres_fe.h" #endif +#include "port/win32ntdll.h" + #include #include #include - -static int -openFlagsToCreateFileFlags(int openFlags) -{ - switch (openFlags & (O_CREAT | O_TRUNC | O_EXCL)) - { - /* O_EXCL is meaningless without O_CREAT */ - case 0: - case O_EXCL: - return OPEN_EXISTING; - - case O_CREAT: - return OPEN_ALWAYS; - - /* O_EXCL is meaningless without O_CREAT */ - case O_TRUNC: - case O_TRUNC | O_EXCL: - return TRUNCATE_EXISTING; - - case O_CREAT | O_TRUNC: - return CREATE_ALWAYS; - - /* O_TRUNC is meaningless with O_CREAT */ - case O_CREAT | O_EXCL: - case O_CREAT | O_TRUNC | O_EXCL: - return CREATE_NEW; - } - - /* will never get here */ - return 0; -} - /* - * - file attribute setting, based on fileMode? + * Internal function used by pgwin32_open() and _pgstat64(). When + * backup_semantics is true, directories may be opened (for limited uses). On + * failure, INVALID_HANDLE_VALUE is returned and errno is set. */ -int -pgwin32_open(const char *fileName, int fileFlags,...) +HANDLE +pgwin32_open_handle(const char *fileName, int fileFlags, bool backup_semantics) { - int fd; HANDLE h = INVALID_HANDLE_VALUE; - SECURITY_ATTRIBUTES sa; int loops = 0; + wchar_t wideFileName[MAX_PATH]; + size_t wideFileNameSize; + wchar_t buffer[MAX_PATH]; + UNICODE_STRING ntFileName; + ACCESS_MASK desiredAccess; + OBJECT_ATTRIBUTES objectAttributes; + IO_STATUS_BLOCK ioStatusBlock; + ULONG fileAttributes; + ULONG shareAccess; + ULONG createDisposition; + ULONG createOptions; + NTSTATUS status; + DWORD err; + + if (initialize_ntdll() < 0) + return INVALID_HANDLE_VALUE; /* Check that we can handle the request */ assert((fileFlags & ((O_RDONLY | O_WRONLY | O_RDWR) | O_APPEND | @@ -72,51 +58,116 @@ pgwin32_open(const char *fileName, int fileFlags,...) _O_SHORT_LIVED | O_DSYNC | O_DIRECT | (O_CREAT | O_TRUNC | O_EXCL) | (O_TEXT | O_BINARY))) == fileFlags); #ifndef FRONTEND - Assert(pgwin32_signal_event != NULL); /* small chance of pg_usleep() */ + /* XXX When called by stat very early on, this fails! */ + //Assert(pgwin32_signal_event != NULL); /* small chance of pg_usleep() */ #endif -#ifdef FRONTEND + /* Convert char string to wchar_t string. */ + wideFileNameSize = MultiByteToWideChar(CP_ACP, 0, fileName, -1, + wideFileName, + lengthof(wideFileName)); + if (wideFileNameSize == 0) + { + _dosmaperr(GetLastError()); + return INVALID_HANDLE_VALUE; + } - /* - * Since PostgreSQL 12, those concurrent-safe versions of open() and - * fopen() can be used by frontends, having as side-effect to switch the - * file-translation mode from O_TEXT to O_BINARY if none is specified. - * Caller may want to enforce the binary or text mode, but if nothing is - * defined make sure that the default mode maps with what versions older - * than 12 have been doing. - */ - if ((fileFlags & O_BINARY) == 0) - fileFlags |= O_TEXT; -#endif + /* Convert DOS/Win32 path to fully qualified NT path. */ + ntFileName.Length = 0; + ntFileName.MaximumLength = sizeof(buffer); + ntFileName.Buffer = buffer; + if (!pg_RtlDosPathNameToNtPathName_U(wideFileName, + &ntFileName, + NULL, + NULL)) + { + /* XXX does GetLastError() work for this function? need the + * _WithStatus version maybe? */ + _dosmaperr(GetLastError()); + return INVALID_HANDLE_VALUE; + } - sa.nLength = sizeof(sa); - sa.bInheritHandle = TRUE; - sa.lpSecurityDescriptor = NULL; - - while ((h = CreateFile(fileName, - /* cannot use O_RDONLY, as it == 0 */ - (fileFlags & O_RDWR) ? (GENERIC_WRITE | GENERIC_READ) : - ((fileFlags & O_WRONLY) ? GENERIC_WRITE : GENERIC_READ), - /* These flags allow concurrent rename/unlink */ - (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE), - &sa, - openFlagsToCreateFileFlags(fileFlags), - FILE_ATTRIBUTE_NORMAL | - ((fileFlags & O_RANDOM) ? FILE_FLAG_RANDOM_ACCESS : 0) | - ((fileFlags & O_SEQUENTIAL) ? FILE_FLAG_SEQUENTIAL_SCAN : 0) | - ((fileFlags & _O_SHORT_LIVED) ? FILE_ATTRIBUTE_TEMPORARY : 0) | - ((fileFlags & O_TEMPORARY) ? FILE_FLAG_DELETE_ON_CLOSE : 0) | - ((fileFlags & O_DIRECT) ? FILE_FLAG_NO_BUFFERING : 0) | - ((fileFlags & O_DSYNC) ? FILE_FLAG_WRITE_THROUGH : 0), - NULL)) == INVALID_HANDLE_VALUE) + /* Convert other parameters. */ + desiredAccess = + FILE_READ_ATTRIBUTES | /* allow fstat() even if O_WRONLY */ + ((fileFlags & O_TEMPORARY) ? DELETE : 0) | + ((fileFlags & O_RDWR) ? FILE_GENERIC_WRITE | FILE_GENERIC_READ : + (fileFlags & O_WRONLY) ? FILE_GENERIC_WRITE : FILE_GENERIC_READ); + shareAccess = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + InitializeObjectAttributes(&objectAttributes, &ntFileName, + OBJ_CASE_INSENSITIVE | OBJ_INHERIT, NULL, NULL); + fileAttributes = + ((fileFlags & _O_SHORT_LIVED) ? FILE_ATTRIBUTE_TEMPORARY : 0); + if (fileAttributes == 0) + fileAttributes = FILE_ATTRIBUTE_NORMAL; + if (fileFlags & O_CREAT) { + if (fileFlags & O_EXCL) + createDisposition = FILE_CREATE; + else if (fileFlags & O_TRUNC) + createDisposition = FILE_OVERWRITE_IF; + else + createDisposition = FILE_OPEN_IF; + } + else if (fileFlags & O_TRUNC) + createDisposition = FILE_OVERWRITE; + else + createDisposition = FILE_OPEN; + createOptions = + FILE_SYNCHRONOUS_IO_NONALERT | + (backup_semantics ? FILE_OPEN_FOR_BACKUP_INTENT : 0) | + ((fileFlags & O_RANDOM) ? FILE_RANDOM_ACCESS : 0) | + ((fileFlags & O_SEQUENTIAL) ? FILE_SEQUENTIAL_ONLY : 0) | + ((fileFlags & O_DIRECT) ? FILE_NO_INTERMEDIATE_BUFFERING : 0) | + ((fileFlags & O_DSYNC) ? FILE_WRITE_THROUGH : 0) | + ((fileFlags & O_TEMPORARY) ? FILE_DELETE_ON_CLOSE : 0); + + for (;;) + { + status = pg_NtCreateFile(&h, + desiredAccess, + &objectAttributes, + &ioStatusBlock, + NULL, + fileAttributes, + shareAccess, + createDisposition, + createOptions, + NULL, + 0); + + if (NT_SUCCESS(status)) + break; + + /* + * Translate STATUS_DELETE_PENDING to a more Unix-like error. + * + * If there's no O_CREAT flag, then we'll pretend the file is + * invisible. With O_CREAT, we have no choice but to report that + * there's a file in the way (which wouldn't happen on Unix). + */ + if (status == STATUS_DELETE_PENDING) + { + if (fileFlags & O_CREAT) + err = ERROR_FILE_EXISTS; + else + err = ERROR_FILE_NOT_FOUND; + _dosmaperr(err); + return INVALID_HANDLE_VALUE; + } + + /* + * For everything else, convert the NT error into a DOS error. This + * is where STATUS_DELETE_PENDING would normally be mapped to + * ERROR_ACCESS_DENIED, and lost to us. + */ + err = pg_RtlNtStatusToDosError(status); + /* * Sharing violation or locking error can indicate antivirus, backup * or similar software that's locking the file. Wait a bit and try * again, giving up after 30 seconds. */ - DWORD err = GetLastError(); - if (err == ERROR_SHARING_VIOLATION || err == ERROR_LOCK_VIOLATION) { @@ -137,41 +188,37 @@ pgwin32_open(const char *fileName, int fileFlags,...) } } - /* - * ERROR_ACCESS_DENIED is returned if the file is deleted but not yet - * gone (Windows NT status code is STATUS_DELETE_PENDING). In that - * case we want to wait a bit and try again, giving up after 1 second - * (since this condition should never persist very long). However, - * there are other commonly-hit cases that return ERROR_ACCESS_DENIED, - * so care is needed. In particular that happens if we try to open a - * directory, or of course if there's an actual file-permissions - * problem. To distinguish these cases, try a stat(). In the - * delete-pending case, it will either also get STATUS_DELETE_PENDING, - * or it will see the file as gone and fail with ENOENT. In other - * cases it will usually succeed. The only somewhat-likely case where - * this coding will uselessly wait is if there's a permissions problem - * with a containing directory, which we hope will never happen in any - * performance-critical code paths. - */ - if (err == ERROR_ACCESS_DENIED) - { - if (loops < 10) - { - struct stat st; - - if (stat(fileName, &st) != 0) - { - pg_usleep(100000); - loops++; - continue; - } - } - } - _dosmaperr(err); - return -1; + return INVALID_HANDLE_VALUE; } + return h; +} + +int +pgwin32_open(const char *fileName, int fileFlags,...) +{ + HANDLE h; + int fd; + + h = pgwin32_open_handle(fileName, fileFlags, false); + if (h == INVALID_HANDLE_VALUE) + return -1; + +#ifdef FRONTEND + + /* + * Since PostgreSQL 12, those concurrent-safe versions of open() and + * fopen() can be used by frontends, having as side-effect to switch the + * file-translation mode from O_TEXT to O_BINARY if none is specified. + * Caller may want to enforce the binary or text mode, but if nothing is + * defined make sure that the default mode maps with what versions older + * than 12 have been doing. + */ + if ((fileFlags & O_BINARY) == 0) + fileFlags |= O_TEXT; +#endif + /* _open_osfhandle will, on error, set errno accordingly */ if ((fd = _open_osfhandle((intptr_t) h, fileFlags & O_APPEND)) < 0) CloseHandle(h); /* will not affect errno */ diff --git a/src/port/win32ntdll.c b/src/port/win32ntdll.c new file mode 100644 index 0000000000..fc9d1b84bb --- /dev/null +++ b/src/port/win32ntdll.c @@ -0,0 +1,68 @@ +/*------------------------------------------------------------------------- + * + * win32ntdll.c + * Dynamically loaded Windows NT functions. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/port/win32ntdll.c + * + *------------------------------------------------------------------------- + */ + +#include "c.h" + +#include "port/win32ntdll.h" + +NtCreateFile_t pg_NtCreateFile; +RtlDosPathNameToNtPathName_U_t pg_RtlDosPathNameToNtPathName_U; +RtlNtStatusToDosError_t pg_RtlNtStatusToDosError; + +int +initialize_ntdll(void) +{ + static bool initialized; + HMODULE module; + + static const struct { + const char *name; + pg_funcptr_t *address; + } routines[] = { + {"NtCreateFile", (pg_funcptr_t *) &pg_NtCreateFile}, + {"RtlDosPathNameToNtPathName_U", + (pg_funcptr_t *) &pg_RtlDosPathNameToNtPathName_U}, + {"RtlNtStatusToDosError", (pg_funcptr_t *) &pg_RtlNtStatusToDosError} + }; + + if (initialized) + return 0; + + if (!(module = LoadLibraryEx("ntdll.dll", NULL, 0))) + { + _dosmaperr(GetLastError()); + return -1; + } + + for (int i = 0; i < lengthof(routines); ++i) + { + pg_funcptr_t address; + + address = (pg_funcptr_t) GetProcAddress(module, routines[i].name); + if (!address) + { + _dosmaperr(GetLastError()); + FreeLibrary(module); + + return -1; + } + + *(pg_funcptr_t *) routines[i].address = address; + } + + initialized = true; + + return 0; +} diff --git a/src/port/win32stat.c b/src/port/win32stat.c index 2ad8ee1359..c851400dc8 100644 --- a/src/port/win32stat.c +++ b/src/port/win32stat.c @@ -18,53 +18,6 @@ #include "c.h" #include -/* - * In order to support MinGW and MSVC2013 we use NtQueryInformationFile as an - * alternative for GetFileInformationByHandleEx. It is loaded from the ntdll - * library. - */ -#if _WIN32_WINNT < 0x0600 -#include - -#if !defined(__MINGW32__) && !defined(__MINGW64__) -/* MinGW includes this in , but it is missing in MSVC */ -typedef struct _FILE_STANDARD_INFORMATION -{ - LARGE_INTEGER AllocationSize; - LARGE_INTEGER EndOfFile; - ULONG NumberOfLinks; - BOOLEAN DeletePending; - BOOLEAN Directory; -} FILE_STANDARD_INFORMATION; -#define FileStandardInformation 5 -#endif /* !defined(__MINGW32__) && - * !defined(__MINGW64__) */ - -typedef NTSTATUS (NTAPI * PFN_NTQUERYINFORMATIONFILE) - (IN HANDLE FileHandle, - OUT PIO_STATUS_BLOCK IoStatusBlock, - OUT PVOID FileInformation, - IN ULONG Length, - IN FILE_INFORMATION_CLASS FileInformationClass); - -static PFN_NTQUERYINFORMATIONFILE _NtQueryInformationFile = NULL; - -static HMODULE ntdll = NULL; - -/* - * Load DLL file just once regardless of how many functions we load/call in it. - */ -static void -LoadNtdll(void) -{ - if (ntdll != NULL) - return; - ntdll = LoadLibraryEx("ntdll.dll", NULL, 0); -} - -#endif /* _WIN32_WINNT < 0x0600 */ - - /* * Convert a FILETIME struct into a 64 bit time_t. */ @@ -162,120 +115,18 @@ int _pgstat64(const char *name, struct stat *buf) { /* - * We must use a handle so lstat() returns the information of the target - * file. To have a reliable test for ERROR_DELETE_PENDING, we use - * NtQueryInformationFile from Windows 2000 or - * GetFileInformationByHandleEx from Server 2008 / Vista. + * Our open wrapper will report STATUS_DELETE_PENDING as ENOENT. We + * request FILE_FLAG_BACKUP_SEMANTICS so that we can open directories too, + * for limited purposes. We use the private handle-based version, so we + * don't risk running out of fds. */ - SECURITY_ATTRIBUTES sa; HANDLE hFile; int ret; -#if _WIN32_WINNT < 0x0600 - IO_STATUS_BLOCK ioStatus; - FILE_STANDARD_INFORMATION standardInfo; -#else - FILE_STANDARD_INFO standardInfo; -#endif - - if (name == NULL || buf == NULL) - { - errno = EINVAL; - return -1; - } - /* fast not-exists check */ - if (GetFileAttributes(name) == INVALID_FILE_ATTRIBUTES) - { - _dosmaperr(GetLastError()); - return -1; - } - - /* get a file handle as lightweight as we can */ - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.bInheritHandle = TRUE; - sa.lpSecurityDescriptor = NULL; - hFile = CreateFile(name, - GENERIC_READ, - (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE), - &sa, - OPEN_EXISTING, - (FILE_FLAG_NO_BUFFERING | FILE_FLAG_BACKUP_SEMANTICS | - FILE_FLAG_OVERLAPPED), - NULL); + hFile = pgwin32_open_handle(name, O_RDONLY, true); if (hFile == INVALID_HANDLE_VALUE) - { - DWORD err = GetLastError(); - - CloseHandle(hFile); - _dosmaperr(err); return -1; - } - - memset(&standardInfo, 0, sizeof(standardInfo)); - -#if _WIN32_WINNT < 0x0600 - if (_NtQueryInformationFile == NULL) - { - /* First time through: load ntdll.dll and find NtQueryInformationFile */ - LoadNtdll(); - if (ntdll == NULL) - { - DWORD err = GetLastError(); - - CloseHandle(hFile); - _dosmaperr(err); - return -1; - } - - _NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE) (pg_funcptr_t) - GetProcAddress(ntdll, "NtQueryInformationFile"); - if (_NtQueryInformationFile == NULL) - { - DWORD err = GetLastError(); - CloseHandle(hFile); - _dosmaperr(err); - return -1; - } - } - - if (!NT_SUCCESS(_NtQueryInformationFile(hFile, &ioStatus, &standardInfo, - sizeof(standardInfo), - FileStandardInformation))) - { - DWORD err = GetLastError(); - - CloseHandle(hFile); - _dosmaperr(err); - return -1; - } -#else - if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo, - sizeof(standardInfo))) - { - DWORD err = GetLastError(); - - CloseHandle(hFile); - _dosmaperr(err); - return -1; - } -#endif /* _WIN32_WINNT < 0x0600 */ - - if (standardInfo.DeletePending) - { - /* - * File has been deleted, but is not gone from the filesystem yet. - * This can happen when some process with FILE_SHARE_DELETE has it - * open, and it will be fully removed once that handle is closed. - * Meanwhile, we can't open it, so indicate that the file just doesn't - * exist. - */ - CloseHandle(hFile); - errno = ENOENT; - return -1; - } - - /* At last we can invoke fileinfo_to_stat */ ret = fileinfo_to_stat(hFile, buf); CloseHandle(hFile); @@ -296,11 +147,6 @@ _pgfstat64(int fileno, struct stat *buf) return -1; } - /* - * Since we already have a file handle there is no need to check for - * ERROR_DELETE_PENDING. - */ - return fileinfo_to_stat(hFile, buf); } diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 84f15f7e85..bebb0578dc 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -107,7 +107,8 @@ sub mkvcbuild pg_strong_random.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c pqsignal.c mkdtemp.c qsort.c qsort_arg.c bsearch_arg.c quotes.c system.c strerror.c tar.c thread.c - win32env.c win32error.c win32security.c win32setlocale.c win32stat.c); + win32env.c win32error.c win32ntdll.c + win32security.c win32setlocale.c win32stat.c); push(@pgportfiles, 'strtof.c') if ($vsVersion < '14.00'); -- 2.30.2