diff --git a/src/port/pg_strong_random.c b/src/port/pg_strong_random.c
index 6d3aa38..34cab6f 100644
--- a/src/port/pg_strong_random.c
+++ b/src/port/pg_strong_random.c
@@ -31,14 +31,6 @@
 #include <Wincrypt.h>
 #endif
 
-#ifdef WIN32
-/*
- * Cache a global crypto provider that only gets freed when the process
- * exits, in case we need random numbers more than once.
- */
-static HCRYPTPROV hProvider = 0;
-#endif
-
 #if defined(USE_DEV_URANDOM)
 /*
  * Read (random) bytes from a file.
@@ -111,28 +103,36 @@ pg_strong_random(void *buf, size_t len)
 	 * Windows has CryptoAPI for strong cryptographic numbers.
 	 */
 #elif defined(USE_WIN32_RANDOM)
-	if (hProvider == 0)
+	HCRYPTPROV	hProvider;
+	DWORD		flags;
+
+	flags = CRYPT_VERIFYCONTEXT | CRYPT_SILENT | CRYPT_MACHINE_KEYSET;
+
+	/* Create a crypto provider */
+	if (!CryptAcquireContext(&hProvider,
+							 NULL,
+							 MS_DEF_PROV,
+							 PROV_RSA_FULL,
+							 flags))
 	{
+		/* If previous creation failed, try with a new key container */
+		flags |= CRYPT_NEWKEYSET;
 		if (!CryptAcquireContext(&hProvider,
 								 NULL,
 								 MS_DEF_PROV,
 								 PROV_RSA_FULL,
-								 CRYPT_VERIFYCONTEXT | CRYPT_SILENT))
-		{
-			/*
-			 * On failure, set back to 0 in case the value was for some reason
-			 * modified.
-			 */
-			hProvider = 0;
-		}
+								 flags))
+			return false;
 	}
-	/* Re-check in case we just retrieved the provider */
-	if (hProvider != 0)
+
+	if (!CryptGenRandom(hProvider, len, buf))
 	{
-		if (CryptGenRandom(hProvider, len, buf))
-			return true;
+		CryptReleaseContext(hProvider, 0);
+		return false;
 	}
-	return false;
+
+	CryptReleaseContext(hProvider, 0);
+	return true;
 
 	/*
 	 * Read /dev/urandom ourselves.