[PATCH] pg_stat_activity: make slow/hanging authentication more visible

From 014f42c62659dbf302b85b9265ab0d6b081b08b3 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Mon, 8 Jul 2024 10:11:56 -0700
Subject: [PATCH v2 1/4] BackgroundPsql: handle empty query results

There won't be a newline at the end of an empty query result. (Before
this fix, the $banner showed up in the result, leading to confusing
debugging sessions.)

recovery/t/037_invalid_database was relying on the non-empty query
results, so I have switched those cases to use "bare" calls to
query_safe() instead.
---
 src/test/perl/PostgreSQL/Test/BackgroundPsql.pm |  2 +-
 src/test/recovery/t/037_invalid_database.pl     | 12 ++++--------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index 3c2aca1c5d..2760e4bc8d 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -223,7 +223,7 @@ sub query
 	$output = $self->{stdout};
 
 	# remove banner again, our caller doesn't care
-	$output =~ s/\n$banner\n$//s;
+	$output =~ s/\n?$banner\n$//s;
 
 	# clear out output for the next query
 	$self->{stdout} = '';
diff --git a/src/test/recovery/t/037_invalid_database.pl b/src/test/recovery/t/037_invalid_database.pl
index 47f524be4c..954c3684a9 100644
--- a/src/test/recovery/t/037_invalid_database.pl
+++ b/src/test/recovery/t/037_invalid_database.pl
@@ -93,13 +93,12 @@ my $bgpsql = $node->background_psql('postgres', on_error_stop => 0);
 my $pid = $bgpsql->query('SELECT pg_backend_pid()');
 
 # create the database, prevent drop database via lock held by a 2PC transaction
-ok( $bgpsql->query_safe(
+$bgpsql->query_safe(
 		qq(
   CREATE DATABASE regression_invalid_interrupt;
   BEGIN;
   LOCK pg_tablespace;
-  PREPARE TRANSACTION 'lock_tblspc';)),
-	"blocked DROP DATABASE completion");
+  PREPARE TRANSACTION 'lock_tblspc';));
 
 # Try to drop. This will wait due to the still held lock.
 $bgpsql->query_until(qr//, "DROP DATABASE regression_invalid_interrupt;\n");
@@ -126,11 +125,8 @@ is($node->psql('regression_invalid_interrupt', ''),
 
 # To properly drop the database, we need to release the lock previously preventing
 # doing so.
-ok($bgpsql->query_safe(qq(ROLLBACK PREPARED 'lock_tblspc')),
-	"unblock DROP DATABASE");
-
-ok($bgpsql->query(qq(DROP DATABASE regression_invalid_interrupt)),
-	"DROP DATABASE invalid_interrupt");
+$bgpsql->query_safe(qq(ROLLBACK PREPARED 'lock_tblspc'));
+$bgpsql->query_safe(qq(DROP DATABASE regression_invalid_interrupt));
 
 $bgpsql->quit();
 
-- 
2.34.1

From 592ef05d30505c4c7a1a84084f80c807fce57476 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Mon, 8 Jul 2024 10:46:55 -0700
Subject: [PATCH v2 2/4] Test::Cluster: let background_psql() work
 asynchronously

Specifying `wait => 0` as a parameter to background_psql() causes it to
return immediately, which lets the client run code during connection.
(This is useful if, for example, connections are blocked on injected
waitpoints.) Clients later call ->wait_connect() manually to complete
the asynchronous connection.
---
 .../perl/PostgreSQL/Test/BackgroundPsql.pm    | 23 ++++++++++++++-----
 src/test/perl/PostgreSQL/Test/Cluster.pm      | 10 +++++++-
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index 2760e4bc8d..13489ee95e 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -81,7 +81,7 @@ string. For C<interactive> sessions, IO::Pty is required.
 sub new
 {
 	my $class = shift;
-	my ($interactive, $psql_params, $timeout) = @_;
+	my ($interactive, $psql_params, $timeout, $wait) = @_;
 	my $psql = {
 		'stdin' => '',
 		'stdout' => '',
@@ -119,14 +119,25 @@ sub new
 
 	my $self = bless $psql, $class;
 
-	$self->_wait_connect();
+	$wait = 1 unless defined($wait);
+	if ($wait)
+	{
+		$self->wait_connect();
+	}
 
 	return $self;
 }
 
-# Internal routine for awaiting psql starting up and being ready to consume
-# input.
-sub _wait_connect
+=pod
+
+=item $session->wait_connect
+
+Returns once psql has started up and is ready to consume input.  This is called
+automatically for clients unless requested otherwise in the constructor.
+
+=cut
+
+sub wait_connect
 {
 	my ($self) = @_;
 
@@ -187,7 +198,7 @@ sub reconnect_and_clear
 	$self->{stdin} = '';
 	$self->{stdout} = '';
 
-	$self->_wait_connect();
+	$self->wait_connect();
 }
 
 =pod
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 0135c5a795..759c9d93c2 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2182,6 +2182,12 @@ connection.
 
 If given, it must be an array reference containing additional parameters to B<psql>.
 
+=item wait => 1
+
+By default, this method will not return until connection has completed (or
+failed).  Set B<wait> to 0 to return immediately instead.  (Clients can call the
+session's C<wait_connect> method manually when needed.)
+
 =back
 
 =cut
@@ -2205,13 +2211,15 @@ sub background_psql
 		'-');
 
 	$params{on_error_stop} = 1 unless defined $params{on_error_stop};
+	$params{wait} = 1 unless defined $params{wait};
 	$timeout = $params{timeout} if defined $params{timeout};
 
 	push @psql_params, '-v', 'ON_ERROR_STOP=1' if $params{on_error_stop};
 	push @psql_params, @{ $params{extra_params} }
 	  if defined $params{extra_params};
 
-	return PostgreSQL::Test::BackgroundPsql->new(0, \@psql_params, $timeout);
+	return PostgreSQL::Test::BackgroundPsql->new(0, \@psql_params, $timeout,
+		$params{wait});
 }
 
 =pod
-- 
2.34.1

From 059f856892fb9e9f75d8ce36bad6533457b0711d Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v2 4/4] WIP: report external auth calls as wait events

Introduce new WAIT_EVENT_AUTHN_* types for various external
authentication systems, to make it obvious what's going wrong if one of
those systems hangs.

TODO:
- don't abuse the IPC wait event group like this
- test
---
 src/backend/libpq/auth.c                      | 54 +++++++++++++++----
 .../utils/activity/wait_event_names.txt       |  5 ++
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 2b607c5270..bda80e88f9 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -1000,6 +1001,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_GSSAPI);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1011,6 +1013,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1222,6 +1225,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1231,6 +1235,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1296,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1305,6 +1312,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1402,19 +1410,25 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
 	if (!port->hba->compat_realm)
 	{
-		int			status = pg_SSPI_make_upn(accountname, sizeof(accountname),
-											  domainname, sizeof(domainname),
-											  port->hba->upn_username);
+		int			status;
+
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
+		status = pg_SSPI_make_upn(accountname, sizeof(accountname),
+								  domainname, sizeof(domainname),
+								  port->hba->upn_username);
+		pgstat_report_wait_end();
 
 		if (status != STATUS_OK)
 			/* Error already reported from pg_SSPI_make_upn */
@@ -2114,7 +2128,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_PAM);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2127,7 +2143,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_PAM);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2478,7 +2496,11 @@ CheckLDAPAuth(Port *port)
 	if (passwd == NULL)
 		return STATUS_EOF;		/* client wouldn't send password */
 
-	if (InitializeLDAPConnection(port, &ldap) == STATUS_ERROR)
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
+	r = InitializeLDAPConnection(port, &ldap);
+	pgstat_report_wait_end();
+
+	if (r == STATUS_ERROR)
 	{
 		/* Error message already sent */
 		pfree(passwd);
@@ -2525,9 +2547,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2550,6 +2575,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2557,6 +2584,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2625,7 +2653,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -2885,12 +2915,16 @@ CheckRADIUSAuth(Port *port)
 	identifiers = list_head(port->hba->radiusidentifiers);
 	foreach(server, port->hba->radiusservers)
 	{
-		int			ret = PerformRadiusTransaction(lfirst(server),
-												   lfirst(secrets),
-												   radiusports ? lfirst(radiusports) : NULL,
-												   identifiers ? lfirst(identifiers) : NULL,
-												   port->user_name,
-												   passwd);
+		int			ret;
+
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_RADIUS);
+		ret = PerformRadiusTransaction(lfirst(server),
+									   lfirst(secrets),
+									   radiusports ? lfirst(radiusports) : NULL,
+									   identifiers ? lfirst(identifiers) : NULL,
+									   port->user_name,
+									   passwd);
+		pgstat_report_wait_end();
 
 		/*------
 		 * STATUS_OK = Login OK
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index db37beeaae..3edbfe4473 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -104,6 +104,11 @@ Section: ClassName - WaitEventIPC
 APPEND_READY	"Waiting for subplan nodes of an <literal>Append</literal> plan node to be ready."
 ARCHIVE_CLEANUP_COMMAND	"Waiting for <xref linkend="guc-archive-cleanup-command"/> to complete."
 ARCHIVE_COMMAND	"Waiting for <xref linkend="guc-archive-command"/> to complete."
+AUTHN_GSSAPI	"Waiting for a response from a Kerberos server via GSSAPI."
+AUTHN_LDAP	"Waiting for a response from an LDAP server."
+AUTHN_PAM	"Waiting for a response from the local PAM service."
+AUTHN_RADIUS	"Waiting for a response from a RADIUS server."
+AUTHN_SSPI	"Waiting for a response from a Windows security provider via SSPI."
 BACKEND_TERMINATION	"Waiting for the termination of another backend."
 BACKUP_WAIT_WAL_ARCHIVE	"Waiting for WAL files required for a backup to be successfully archived."
 BGWORKER_SHUTDOWN	"Waiting for background worker to shut down."
-- 
2.34.1

From 22ce00adebdd242e747036d46e1904307287feaa Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v2 3/4] pgstat: report in earlier with STATE_AUTHENTICATING

Add pgstat_bestart_pre_auth(), which reports an 'authenticating' state
while waiting for client authentication to complete. Since we hold a
transaction open across that call, and some authentication methods call
out to external systems, having a pg_stat_activity entry helps DBAs
debug when things go badly wrong.
---
 src/backend/utils/activity/backend_status.c   | 37 +++++++++-
 src/backend/utils/adt/pgstatfuncs.c           |  3 +
 src/backend/utils/init/postinit.c             | 11 +++
 src/include/utils/backend_status.h            |  2 +
 src/test/authentication/Makefile              |  2 +
 src/test/authentication/meson.build           |  4 +
 .../authentication/t/007_injection_points.pl  | 73 +++++++++++++++++++
 7 files changed, 128 insertions(+), 4 deletions(-)
 create mode 100644 src/test/authentication/t/007_injection_points.pl

diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 1ccf4c6d83..c996049bbe 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -71,6 +71,7 @@ static int	localNumBackends = 0;
 static MemoryContext backendStatusSnapContext;
 
 
+static void pgstat_bestart_internal(bool pre_auth);
 static void pgstat_beshutdown_hook(int code, Datum arg);
 static void pgstat_read_current_status(void);
 static void pgstat_setup_backend_status_context(void);
@@ -271,6 +272,34 @@ pgstat_beinit(void)
  */
 void
 pgstat_bestart(void)
+{
+	pgstat_bestart_internal(false);
+}
+
+
+/* ----------
+ * pgstat_bestart_pre_auth() -
+ *
+ *	Like pgstat_beinit(), above, but it's designed to be called before
+ *	authentication has been performed (so we have no user or database IDs).
+ *	Called from InitPostgres.
+ *----------
+ */
+void
+pgstat_bestart_pre_auth(void)
+{
+	pgstat_bestart_internal(true);
+}
+
+
+/* ----------
+ * pgstat_bestart_internal() -
+ *
+ *	Implementation of both flavors of pgstat_bestart().
+ *----------
+ */
+static void
+pgstat_bestart_internal(bool pre_auth)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
@@ -320,9 +349,9 @@ pgstat_bestart(void)
 	lbeentry.st_databaseid = MyDatabaseId;
 
 	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
+	if (!pre_auth && (lbeentry.st_backendType == B_BACKEND
+					  || lbeentry.st_backendType == B_WAL_SENDER
+					  || lbeentry.st_backendType == B_BG_WORKER))
 		lbeentry.st_userid = GetSessionUserId();
 	else
 		lbeentry.st_userid = InvalidOid;
@@ -377,7 +406,7 @@ pgstat_bestart(void)
 	lbeentry.st_gss = false;
 #endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = pre_auth ? STATE_AUTHENTICATING : STATE_UNDEFINED;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 3876339ee1..f34e4a1643 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -366,6 +366,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_AUTHENTICATING:
+					values[4] = CStringGetTextDatum("authenticating");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index 25867c8bd5..83da2790b2 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -878,6 +879,16 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
+		/*
+		 * Authentication can take a while, during which time we're holding a
+		 * transaction open. Fill in enough of a backend status so that DBAs can
+		 * observe what's going on. (The later call to pgstat_bestart() will
+		 * fill in the rest of the status after we've authenticated.)
+		 */
+		pgstat_bestart_pre_auth();
+		INJECTION_POINT("init-pre-auth");
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 7b7f6f59d0..f673c6a6ac 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_AUTHENTICATING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,6 +310,7 @@ extern void CreateSharedBackendStatus(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
+extern void pgstat_bestart_pre_auth(void);
 extern void pgstat_bestart(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index da0b71873a..7bbc3b6e57 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,8 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index 8f5688dcc1..09bad8f2b8 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_injection_points.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_injection_points.pl b/src/test/authentication/t/007_injection_points.pl
new file mode 100644
index 0000000000..96ed691d93
--- /dev/null
+++ b/src/test/authentication/t/007_injection_points.pl
@@ -0,0 +1,73 @@
+
+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
+
+# Tests requiring injection_points functionality, to check on behavior that
+# would otherwise race against authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang in authentication. Use the
+# $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'authenticating';");
+	last if $pid ne "";
+
+	usleep(500_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state = $psql->query(
+		"SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(500_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
-- 
2.34.1

From 7f404f5ee8aad2a4286d5251a2712ce978ee3fc2 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Mon, 8 Jul 2024 10:46:55 -0700
Subject: [PATCH v3 2/4] Test::Cluster: let background_psql() work
 asynchronously

Specifying `wait => 0` as a parameter to background_psql() causes it to
return immediately, which lets the client run code during connection.
(This is useful if, for example, connections are blocked on injected
waitpoints.) Clients later call ->wait_connect() manually to complete
the asynchronous connection.
---
 .../perl/PostgreSQL/Test/BackgroundPsql.pm    | 23 ++++++++++++++-----
 src/test/perl/PostgreSQL/Test/Cluster.pm      | 10 +++++++-
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index 2760e4bc8d..13489ee95e 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -81,7 +81,7 @@ string. For C<interactive> sessions, IO::Pty is required.
 sub new
 {
 	my $class = shift;
-	my ($interactive, $psql_params, $timeout) = @_;
+	my ($interactive, $psql_params, $timeout, $wait) = @_;
 	my $psql = {
 		'stdin' => '',
 		'stdout' => '',
@@ -119,14 +119,25 @@ sub new
 
 	my $self = bless $psql, $class;
 
-	$self->_wait_connect();
+	$wait = 1 unless defined($wait);
+	if ($wait)
+	{
+		$self->wait_connect();
+	}
 
 	return $self;
 }
 
-# Internal routine for awaiting psql starting up and being ready to consume
-# input.
-sub _wait_connect
+=pod
+
+=item $session->wait_connect
+
+Returns once psql has started up and is ready to consume input.  This is called
+automatically for clients unless requested otherwise in the constructor.
+
+=cut
+
+sub wait_connect
 {
 	my ($self) = @_;
 
@@ -187,7 +198,7 @@ sub reconnect_and_clear
 	$self->{stdin} = '';
 	$self->{stdout} = '';
 
-	$self->_wait_connect();
+	$self->wait_connect();
 }
 
 =pod
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index fe6ebf10f7..134ddfaa70 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2191,6 +2191,12 @@ connection.
 
 If given, it must be an array reference containing additional parameters to B<psql>.
 
+=item wait => 1
+
+By default, this method will not return until connection has completed (or
+failed).  Set B<wait> to 0 to return immediately instead.  (Clients can call the
+session's C<wait_connect> method manually when needed.)
+
 =back
 
 =cut
@@ -2214,13 +2220,15 @@ sub background_psql
 		'-');
 
 	$params{on_error_stop} = 1 unless defined $params{on_error_stop};
+	$params{wait} = 1 unless defined $params{wait};
 	$timeout = $params{timeout} if defined $params{timeout};
 
 	push @psql_params, '-v', 'ON_ERROR_STOP=1' if $params{on_error_stop};
 	push @psql_params, @{ $params{extra_params} }
 	  if defined $params{extra_params};
 
-	return PostgreSQL::Test::BackgroundPsql->new(0, \@psql_params, $timeout);
+	return PostgreSQL::Test::BackgroundPsql->new(0, \@psql_params, $timeout,
+		$params{wait});
 }
 
 =pod
-- 
2.34.1

From 8f9315949e997e24f68e41d7450f7883f46bf54e Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Mon, 8 Jul 2024 10:11:56 -0700
Subject: [PATCH v3 1/4] BackgroundPsql: handle empty query results

There won't be a newline at the end of an empty query result. (Before
this fix, the $banner showed up in the result, leading to confusing
debugging sessions.)

recovery/t/037_invalid_database was relying on the non-empty query
results, so I have switched those cases to use "bare" calls to
query_safe() instead.
---
 src/test/perl/PostgreSQL/Test/BackgroundPsql.pm |  2 +-
 src/test/recovery/t/037_invalid_database.pl     | 12 ++++--------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index 3c2aca1c5d..2760e4bc8d 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -223,7 +223,7 @@ sub query
 	$output = $self->{stdout};
 
 	# remove banner again, our caller doesn't care
-	$output =~ s/\n$banner\n$//s;
+	$output =~ s/\n?$banner\n$//s;
 
 	# clear out output for the next query
 	$self->{stdout} = '';
diff --git a/src/test/recovery/t/037_invalid_database.pl b/src/test/recovery/t/037_invalid_database.pl
index 47f524be4c..954c3684a9 100644
--- a/src/test/recovery/t/037_invalid_database.pl
+++ b/src/test/recovery/t/037_invalid_database.pl
@@ -93,13 +93,12 @@ my $bgpsql = $node->background_psql('postgres', on_error_stop => 0);
 my $pid = $bgpsql->query('SELECT pg_backend_pid()');
 
 # create the database, prevent drop database via lock held by a 2PC transaction
-ok( $bgpsql->query_safe(
+$bgpsql->query_safe(
 		qq(
   CREATE DATABASE regression_invalid_interrupt;
   BEGIN;
   LOCK pg_tablespace;
-  PREPARE TRANSACTION 'lock_tblspc';)),
-	"blocked DROP DATABASE completion");
+  PREPARE TRANSACTION 'lock_tblspc';));
 
 # Try to drop. This will wait due to the still held lock.
 $bgpsql->query_until(qr//, "DROP DATABASE regression_invalid_interrupt;\n");
@@ -126,11 +125,8 @@ is($node->psql('regression_invalid_interrupt', ''),
 
 # To properly drop the database, we need to release the lock previously preventing
 # doing so.
-ok($bgpsql->query_safe(qq(ROLLBACK PREPARED 'lock_tblspc')),
-	"unblock DROP DATABASE");
-
-ok($bgpsql->query(qq(DROP DATABASE regression_invalid_interrupt)),
-	"DROP DATABASE invalid_interrupt");
+$bgpsql->query_safe(qq(ROLLBACK PREPARED 'lock_tblspc'));
+$bgpsql->query_safe(qq(DROP DATABASE regression_invalid_interrupt));
 
 $bgpsql->quit();
 
-- 
2.34.1

From a7a7551d09ccbd90bbbc26b75e3be718ec6a085e Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v3 3/4] pgstat: report in earlier with STATE_AUTHENTICATING

Add pgstat_bestart_pre_auth(), which reports an 'authenticating' state
while waiting for backend initialization and client authentication to
complete. Since we hold a transaction open for a good amount of that,
and some authentication methods call out to external systems, having a
pg_stat_activity entry helps DBAs debug when things go badly wrong.
---
 src/backend/utils/activity/backend_status.c   | 37 +++++++++-
 src/backend/utils/adt/pgstatfuncs.c           |  3 +
 src/backend/utils/init/postinit.c             | 20 ++++-
 src/include/utils/backend_status.h            |  2 +
 src/test/authentication/Makefile              |  2 +
 src/test/authentication/meson.build           |  4 +
 .../authentication/t/007_injection_points.pl  | 73 +++++++++++++++++++
 7 files changed, 134 insertions(+), 7 deletions(-)
 create mode 100644 src/test/authentication/t/007_injection_points.pl

diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 34a55e2177..d693b2dff7 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -71,6 +71,7 @@ static int	localNumBackends = 0;
 static MemoryContext backendStatusSnapContext;
 
 
+static void pgstat_bestart_internal(bool pre_auth);
 static void pgstat_beshutdown_hook(int code, Datum arg);
 static void pgstat_read_current_status(void);
 static void pgstat_setup_backend_status_context(void);
@@ -271,6 +272,34 @@ pgstat_beinit(void)
  */
 void
 pgstat_bestart(void)
+{
+	pgstat_bestart_internal(false);
+}
+
+
+/* ----------
+ * pgstat_bestart_pre_auth() -
+ *
+ *	Like pgstat_beinit(), above, but it's designed to be called before
+ *	authentication has been performed (so we have no user or database IDs).
+ *	Called from InitPostgres.
+ *----------
+ */
+void
+pgstat_bestart_pre_auth(void)
+{
+	pgstat_bestart_internal(true);
+}
+
+
+/* ----------
+ * pgstat_bestart_internal() -
+ *
+ *	Implementation of both flavors of pgstat_bestart().
+ *----------
+ */
+static void
+pgstat_bestart_internal(bool pre_auth)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
@@ -320,9 +349,9 @@ pgstat_bestart(void)
 	lbeentry.st_databaseid = MyDatabaseId;
 
 	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
+	if (!pre_auth && (lbeentry.st_backendType == B_BACKEND
+					  || lbeentry.st_backendType == B_WAL_SENDER
+					  || lbeentry.st_backendType == B_BG_WORKER))
 		lbeentry.st_userid = GetSessionUserId();
 	else
 		lbeentry.st_userid = InvalidOid;
@@ -377,7 +406,7 @@ pgstat_bestart(void)
 	lbeentry.st_gss = false;
 #endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = pre_auth ? STATE_AUTHENTICATING : STATE_UNDEFINED;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 3221137123..b007f0d5a1 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -366,6 +366,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_AUTHENTICATING:
+					values[4] = CStringGetTextDatum("authenticating");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index 3b50ce19a2..e2034829e4 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -680,6 +681,21 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That way,
+	 * if LWLocks or third-party authentication should happen to hang, the DBA
+	 * will still be able to see what's going on. (A later call to
+	 * pgstat_bestart() will fill in the rest of the status.)
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_pre_auth();
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -748,9 +764,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -848,6 +861,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 97874300c3..18f79bd3ad 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_AUTHENTICATING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,6 +310,7 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
+extern void pgstat_bestart_pre_auth(void);
 extern void pgstat_bestart(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index da0b71873a..7bbc3b6e57 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,8 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index 8f5688dcc1..09bad8f2b8 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_injection_points.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_injection_points.pl b/src/test/authentication/t/007_injection_points.pl
new file mode 100644
index 0000000000..96ed691d93
--- /dev/null
+++ b/src/test/authentication/t/007_injection_points.pl
@@ -0,0 +1,73 @@
+
+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
+
+# Tests requiring injection_points functionality, to check on behavior that
+# would otherwise race against authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang in authentication. Use the
+# $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'authenticating';");
+	last if $pid ne "";
+
+	usleep(500_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state = $psql->query(
+		"SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(500_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
-- 
2.34.1

From 5c85c3e0e99cfb2beaa4f0e0a4a7e1cc4ab91af3 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v3 4/4] WIP: report external auth calls as wait events

Introduce new WAIT_EVENT_AUTHN_* types for various external
authentication systems, to make it obvious what's going wrong if one of
those systems hangs.

TODO:
- don't abuse the IPC wait event group like this
- test
---
 src/backend/libpq/auth.c                      | 54 +++++++++++++++----
 .../utils/activity/wait_event_names.txt       |  5 ++
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 2b607c5270..bda80e88f9 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -1000,6 +1001,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_GSSAPI);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1011,6 +1013,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1222,6 +1225,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1231,6 +1235,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1296,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1305,6 +1312,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1402,19 +1410,25 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
 	if (!port->hba->compat_realm)
 	{
-		int			status = pg_SSPI_make_upn(accountname, sizeof(accountname),
-											  domainname, sizeof(domainname),
-											  port->hba->upn_username);
+		int			status;
+
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
+		status = pg_SSPI_make_upn(accountname, sizeof(accountname),
+								  domainname, sizeof(domainname),
+								  port->hba->upn_username);
+		pgstat_report_wait_end();
 
 		if (status != STATUS_OK)
 			/* Error already reported from pg_SSPI_make_upn */
@@ -2114,7 +2128,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_PAM);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2127,7 +2143,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_PAM);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2478,7 +2496,11 @@ CheckLDAPAuth(Port *port)
 	if (passwd == NULL)
 		return STATUS_EOF;		/* client wouldn't send password */
 
-	if (InitializeLDAPConnection(port, &ldap) == STATUS_ERROR)
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
+	r = InitializeLDAPConnection(port, &ldap);
+	pgstat_report_wait_end();
+
+	if (r == STATUS_ERROR)
 	{
 		/* Error message already sent */
 		pfree(passwd);
@@ -2525,9 +2547,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2550,6 +2575,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2557,6 +2584,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2625,7 +2653,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -2885,12 +2915,16 @@ CheckRADIUSAuth(Port *port)
 	identifiers = list_head(port->hba->radiusidentifiers);
 	foreach(server, port->hba->radiusservers)
 	{
-		int			ret = PerformRadiusTransaction(lfirst(server),
-												   lfirst(secrets),
-												   radiusports ? lfirst(radiusports) : NULL,
-												   identifiers ? lfirst(identifiers) : NULL,
-												   port->user_name,
-												   passwd);
+		int			ret;
+
+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_RADIUS);
+		ret = PerformRadiusTransaction(lfirst(server),
+									   lfirst(secrets),
+									   radiusports ? lfirst(radiusports) : NULL,
+									   identifiers ? lfirst(identifiers) : NULL,
+									   port->user_name,
+									   passwd);
+		pgstat_report_wait_end();
 
 		/*------
 		 * STATUS_OK = Login OK
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index 8efb4044d6..6ce81edc33 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -105,6 +105,11 @@ Section: ClassName - WaitEventIPC
 APPEND_READY	"Waiting for subplan nodes of an <literal>Append</literal> plan node to be ready."
 ARCHIVE_CLEANUP_COMMAND	"Waiting for <xref linkend="guc-archive-cleanup-command"/> to complete."
 ARCHIVE_COMMAND	"Waiting for <xref linkend="guc-archive-command"/> to complete."
+AUTHN_GSSAPI	"Waiting for a response from a Kerberos server via GSSAPI."
+AUTHN_LDAP	"Waiting for a response from an LDAP server."
+AUTHN_PAM	"Waiting for a response from the local PAM service."
+AUTHN_RADIUS	"Waiting for a response from a RADIUS server."
+AUTHN_SSPI	"Waiting for a response from a Windows security provider via SSPI."
 BACKEND_TERMINATION	"Waiting for the termination of another backend."
 BACKUP_WAIT_WAL_ARCHIVE	"Waiting for WAL files required for a backup to be successfully archived."
 BGWORKER_SHUTDOWN	"Waiting for background worker to shut down."
-- 
2.34.1

1:  8f9315949e = 1:  64289b97e5 BackgroundPsql: handle empty query results
2:  7f404f5ee8 ! 2:  18a9531a25 Test::Cluster: let background_psql() work asynchronously
    @@ src/test/perl/PostgreSQL/Test/Cluster.pm: connection.
      
      =cut
     @@ src/test/perl/PostgreSQL/Test/Cluster.pm: sub background_psql
    - 		'-');
    + 		'-XAtq', '-d', $psql_connstr, '-f', '-');
      
      	$params{on_error_stop} = 1 unless defined $params{on_error_stop};
     +	$params{wait} = 1 unless defined $params{wait};
3:  a7a7551d09 ! 3:  c8071f91d8 pgstat: report in earlier with STATE_AUTHENTICATING
    @@ Metadata
     Author: Jacob Champion <jacob.champion@enterprisedb.com>
     
      ## Commit message ##
    -    pgstat: report in earlier with STATE_AUTHENTICATING
    +    pgstat: report in earlier with STATE_STARTING
     
    -    Add pgstat_bestart_pre_auth(), which reports an 'authenticating' state
    -    while waiting for backend initialization and client authentication to
    +    Add pgstat_bestart_pre_auth(), which reports a 'starting' state while
    +    waiting for backend initialization and client authentication to
         complete. Since we hold a transaction open for a good amount of that,
         and some authentication methods call out to external systems, having a
         pg_stat_activity entry helps DBAs debug when things go badly wrong.
     
    + ## doc/src/sgml/monitoring.sgml ##
    +@@ doc/src/sgml/monitoring.sgml: postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
    +        Current overall state of this backend.
    +        Possible values are:
    +        <itemizedlist>
    ++        <listitem>
    ++         <para>
    ++          <literal>starting</literal>: The backend is in initial startup. Client
    ++          authentication is performed during this phase.
    ++         </para>
    ++        </listitem>
    +         <listitem>
    +         <para>
    +           <literal>active</literal>: The backend is executing a query.
    +
      ## src/backend/utils/activity/backend_status.c ##
     @@ src/backend/utils/activity/backend_status.c: static int	localNumBackends = 0;
      static MemoryContext backendStatusSnapContext;
    @@ src/backend/utils/activity/backend_status.c: pgstat_bestart(void)
      #endif
      
     -	lbeentry.st_state = STATE_UNDEFINED;
    -+	lbeentry.st_state = pre_auth ? STATE_AUTHENTICATING : STATE_UNDEFINED;
    ++	lbeentry.st_state = pre_auth ? STATE_STARTING : STATE_UNDEFINED;
      	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
      	lbeentry.st_progress_command_target = InvalidOid;
      	lbeentry.st_query_id = UINT64CONST(0);
    @@ src/backend/utils/adt/pgstatfuncs.c: pg_stat_get_activity(PG_FUNCTION_ARGS)
      
      			switch (beentry->st_state)
      			{
    -+				case STATE_AUTHENTICATING:
    -+					values[4] = CStringGetTextDatum("authenticating");
    ++				case STATE_STARTING:
    ++					values[4] = CStringGetTextDatum("starting");
     +					break;
      				case STATE_IDLE:
      					values[4] = CStringGetTextDatum("idle");
    @@ src/backend/utils/init/postinit.c: InitPostgres(const char *in_dbname, Oid dboid
     +	pgstat_beinit();
     +
     +	/*
    -+	 * This is a convenient time to sketch in a partial pgstat entry. That way,
    -+	 * if LWLocks or third-party authentication should happen to hang, the DBA
    -+	 * will still be able to see what's going on. (A later call to
    ++	 * This is a convenient time to sketch in a partial pgstat entry. That
    ++	 * way, if LWLocks or third-party authentication should happen to hang,
    ++	 * the DBA will still be able to see what's going on. (A later call to
     +	 * pgstat_bestart() will fill in the rest of the status.)
     +	 */
     +	if (!bootstrap)
    @@ src/include/utils/backend_status.h
      typedef enum BackendState
      {
      	STATE_UNDEFINED,
    -+	STATE_AUTHENTICATING,
    ++	STATE_STARTING,
      	STATE_IDLE,
      	STATE_RUNNING,
      	STATE_IDLEINTRANSACTION,
    @@ src/test/authentication/t/007_injection_points.pl (new)
     +my $psql = $node->background_psql('postgres');
     +$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
     +
    -+# From this point on, all new connections will hang in authentication. Use the
    -+# $psql connection handle for server interaction.
    ++# From this point on, all new connections will hang during startup, just before
    ++# authentication. Use the $psql connection handle for server interaction.
     +my $conn = $node->background_psql('postgres', wait => 0);
     +
     +# Wait for the connection to show up.
    @@ src/test/authentication/t/007_injection_points.pl (new)
     +while (1)
     +{
     +	$pid = $psql->query(
    -+		"SELECT pid FROM pg_stat_activity WHERE state = 'authenticating';");
    ++		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
     +	last if $pid ne "";
     +
     +	usleep(500_000);
    @@ src/test/authentication/t/007_injection_points.pl (new)
     +# Make sure the pgstat entry is updated eventually.
     +while (1)
     +{
    -+	my $state = $psql->query(
    -+		"SELECT state FROM pg_stat_activity WHERE pid = $pid;");
    ++	my $state =
    ++	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
     +	last if $state eq "idle";
     +
     +	note "state for backend $pid is '$state'; waiting for 'idle'...";
4:  5c85c3e0e9 ! 4:  d14b97cb77 WIP: report external auth calls as wait events
    @@ Metadata
     Author: Jacob Champion <jacob.champion@enterprisedb.com>
     
      ## Commit message ##
    -    WIP: report external auth calls as wait events
    +    Report external auth calls as wait events
     
    -    Introduce new WAIT_EVENT_AUTHN_* types for various external
    -    authentication systems, to make it obvious what's going wrong if one of
    -    those systems hangs.
    +    Introduce a new "Auth" wait class for various external authentication
    +    systems, to make it obvious what's going wrong if one of those systems
    +    hangs. Each new wait event is unique in order to more easily pinpoint
    +    problematic locations in the code.
     
    -    TODO:
    -    - don't abuse the IPC wait event group like this
    -    - test
    +    Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
    +
    + ## doc/src/sgml/monitoring.sgml ##
    +@@ doc/src/sgml/monitoring.sgml: postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
    +        see <xref linkend="wait-event-activity-table"/>.
    +       </entry>
    +      </row>
    ++     <row>
    ++      <entry><literal>Auth</literal></entry>
    ++      <entry>The server process is waiting for an external system to
    ++       authenticate and/or authorize the client connection.
    ++       <literal>wait_event</literal> will identify the specific wait point;
    ++       see <xref linkend="wait-event-auth-table"/>.
    ++      </entry>
    ++     </row>
    +      <row>
    +       <entry><literal>BufferPin</literal></entry>
    +       <entry>The server process is waiting for exclusive access to
     
      ## src/backend/libpq/auth.c ##
     @@
    @@ src/backend/libpq/auth.c: pg_GSS_recvauth(Port *port)
      		elog(DEBUG4, "processing received GSS token of length %u",
      			 (unsigned int) gbuf.length);
      
    -+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_GSSAPI);
    ++		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
      		maj_stat = gss_accept_sec_context(&min_stat,
      										  &port->gss->ctx,
      										  port->gss->cred,
    @@ src/backend/libpq/auth.c: pg_SSPI_recvauth(Port *port)
      	/*
      	 * Acquire a handle to the server credentials.
      	 */
    -+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
    ++	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
      	r = AcquireCredentialsHandle(NULL,
      								 "negotiate",
      								 SECPKG_CRED_INBOUND,
    @@ src/backend/libpq/auth.c: pg_SSPI_recvauth(Port *port)
      		elog(DEBUG4, "processing received SSPI token of length %u",
      			 (unsigned int) buf.len);
      
    -+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
    ++		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
      		r = AcceptSecurityContext(&sspicred,
      								  sspictx,
      								  &inbuf,
    @@ src/backend/libpq/auth.c: pg_SSPI_recvauth(Port *port)
      
      	CloseHandle(token);
      
    -+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
    ++	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
      	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
      						  domainname, &domainnamesize, &accountnameuse))
      		ereport(ERROR,
    @@ src/backend/libpq/auth.c: pg_SSPI_recvauth(Port *port)
     -											  port->hba->upn_username);
     +		int			status;
     +
    -+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_SSPI);
    ++		pgstat_report_wait_start(WAIT_EVENT_SSPI_MAKE_UPN);
     +		status = pg_SSPI_make_upn(accountname, sizeof(accountname),
     +								  domainname, sizeof(domainname),
     +								  port->hba->upn_username);
    @@ src/backend/libpq/auth.c: CheckPAMAuth(Port *port, const char *user, const char
      		return STATUS_ERROR;
      	}
      
    -+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_PAM);
    ++	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
      	retval = pam_authenticate(pamh, 0);
     +	pgstat_report_wait_end();
      
    @@ src/backend/libpq/auth.c: CheckPAMAuth(Port *port, const char *user, const char
      		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
      	}
      
    -+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_PAM);
    ++	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
      	retval = pam_acct_mgmt(pamh, 0);
     +	pgstat_report_wait_end();
      
    @@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
      		return STATUS_EOF;		/* client wouldn't send password */
      
     -	if (InitializeLDAPConnection(port, &ldap) == STATUS_ERROR)
    -+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
    ++	pgstat_report_wait_start(WAIT_EVENT_LDAP_INITIALIZE);
     +	r = InitializeLDAPConnection(port, &ldap);
     +	pgstat_report_wait_end();
     +
    @@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
      		 * Bind with a pre-defined username/password (if available) for
      		 * searching. If none is specified, this turns into an anonymous bind.
      		 */
    -+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
    ++		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
      		r = ldap_simple_bind_s(ldap,
      							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
      							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
    @@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
      
      		search_message = NULL;
     +
    -+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
    ++		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
      		r = ldap_search_s(ldap,
      						  port->hba->ldapbasedn,
      						  port->hba->ldapscope,
    @@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
      							port->user_name,
      							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
      
    -+	pgstat_report_wait_start(WAIT_EVENT_AUTHN_LDAP);
    ++	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
      	r = ldap_simple_bind_s(ldap, fulluser, passwd);
     +	pgstat_report_wait_end();
      
    @@ src/backend/libpq/auth.c: CheckRADIUSAuth(Port *port)
     -												   passwd);
     +		int			ret;
     +
    -+		pgstat_report_wait_start(WAIT_EVENT_AUTHN_RADIUS);
    ++		pgstat_report_wait_start(WAIT_EVENT_RADIUS_TRANSACTION);
     +		ret = PerformRadiusTransaction(lfirst(server),
     +									   lfirst(secrets),
     +									   radiusports ? lfirst(radiusports) : NULL,
    @@ src/backend/libpq/auth.c: CheckRADIUSAuth(Port *port)
      		/*------
      		 * STATUS_OK = Login OK
     
    + ## src/backend/utils/activity/wait_event.c ##
    +@@ src/backend/utils/activity/wait_event.c: static const char *pgstat_get_wait_client(WaitEventClient w);
    + static const char *pgstat_get_wait_ipc(WaitEventIPC w);
    + static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
    + static const char *pgstat_get_wait_io(WaitEventIO w);
    ++static const char *pgstat_get_wait_auth(WaitEventAuth w);
    + 
    + 
    + static uint32 local_my_wait_event_info;
    +@@ src/backend/utils/activity/wait_event.c: pgstat_get_wait_event_type(uint32 wait_event_info)
    + 		case PG_WAIT_INJECTIONPOINT:
    + 			event_type = "InjectionPoint";
    + 			break;
    ++		case PG_WAIT_AUTH:
    ++			event_type = "Auth";
    ++			break;
    + 		default:
    + 			event_type = "???";
    + 			break;
    +@@ src/backend/utils/activity/wait_event.c: pgstat_get_wait_event(uint32 wait_event_info)
    + 				event_name = pgstat_get_wait_io(w);
    + 				break;
    + 			}
    ++		case PG_WAIT_AUTH:
    ++			{
    ++				WaitEventAuth w = (WaitEventAuth) wait_event_info;
    ++
    ++				event_name = pgstat_get_wait_auth(w);
    ++				break;
    ++			}
    + 		default:
    + 			event_name = "unknown wait event";
    + 			break;
    +
      ## src/backend/utils/activity/wait_event_names.txt ##
    -@@ src/backend/utils/activity/wait_event_names.txt: Section: ClassName - WaitEventIPC
    - APPEND_READY	"Waiting for subplan nodes of an <literal>Append</literal> plan node to be ready."
    - ARCHIVE_CLEANUP_COMMAND	"Waiting for <xref linkend="guc-archive-cleanup-command"/> to complete."
    - ARCHIVE_COMMAND	"Waiting for <xref linkend="guc-archive-command"/> to complete."
    -+AUTHN_GSSAPI	"Waiting for a response from a Kerberos server via GSSAPI."
    -+AUTHN_LDAP	"Waiting for a response from an LDAP server."
    -+AUTHN_PAM	"Waiting for a response from the local PAM service."
    -+AUTHN_RADIUS	"Waiting for a response from a RADIUS server."
    -+AUTHN_SSPI	"Waiting for a response from a Windows security provider via SSPI."
    - BACKEND_TERMINATION	"Waiting for the termination of another backend."
    - BACKUP_WAIT_WAL_ARCHIVE	"Waiting for WAL files required for a backup to be successfully archived."
    - BGWORKER_SHUTDOWN	"Waiting for background worker to shut down."
    +@@ src/backend/utils/activity/wait_event_names.txt: XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
    + 
    + ABI_compatibility:
    + 
    ++#
    ++# Wait Events - Auth
    ++#
    ++# Use this category when a process is waiting for a third party to
    ++# authenticate/authorize the user.
    ++#
    ++
    ++Section: ClassName - WaitEventAuth
    ++
    ++GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
    ++LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
    ++LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
    ++LDAP_INITIALIZE	"Waiting to initialize an LDAP connection."
    ++LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
    ++PAM_ACCT_MGMT	"Waiting for the local PAM service to validate the user account."
    ++PAM_AUTHENTICATE	"Waiting for the local PAM service to authenticate the user."
    ++RADIUS_TRANSACTION	"Waiting for a RADIUS transaction to complete."
    ++SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
    ++SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
    ++SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's account SID."
    ++SSPI_MAKE_UPN	"Waiting for Windows to translate a Kerberos UPN."
    ++
    ++ABI_compatibility:
    ++
    + #
    + # Wait Events - Timeout
    + #
    +
    + ## src/include/utils/wait_event.h ##
    +@@
    + #define PG_WAIT_TIMEOUT				0x09000000U
    + #define PG_WAIT_IO					0x0A000000U
    + #define PG_WAIT_INJECTIONPOINT		0x0B000000U
    ++#define PG_WAIT_AUTH				0x0C000000U
    + 
    + /* enums for wait events */
    + #include "utils/wait_event_types.h"
    +
    + ## src/test/regress/expected/sysviews.out ##
    +@@ src/test/regress/expected/sysviews.out: select type, count(*) > 0 as ok FROM pg_wait_events
    +    type    | ok 
    + -----------+----
    +  Activity  | t
    ++ Auth      | t
    +  BufferPin | t
    +  Client    | t
    +  Extension | t
    +@@ src/test/regress/expected/sysviews.out: select type, count(*) > 0 as ok FROM pg_wait_events
    +  LWLock    | t
    +  Lock      | t
    +  Timeout   | t
    +-(9 rows)
    ++(10 rows)
    + 
    + -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
    + -- more-or-less working.  We can't test their contents in any great detail

From 64289b97e571fe660be57273efec360ba11d96ff Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Mon, 8 Jul 2024 10:11:56 -0700
Subject: [PATCH v4 1/4] BackgroundPsql: handle empty query results

There won't be a newline at the end of an empty query result. (Before
this fix, the $banner showed up in the result, leading to confusing
debugging sessions.)

recovery/t/037_invalid_database was relying on the non-empty query
results, so I have switched those cases to use "bare" calls to
query_safe() instead.
---
 src/test/perl/PostgreSQL/Test/BackgroundPsql.pm |  2 +-
 src/test/recovery/t/037_invalid_database.pl     | 12 ++++--------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index 3c2aca1c5d..2760e4bc8d 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -223,7 +223,7 @@ sub query
 	$output = $self->{stdout};
 
 	# remove banner again, our caller doesn't care
-	$output =~ s/\n$banner\n$//s;
+	$output =~ s/\n?$banner\n$//s;
 
 	# clear out output for the next query
 	$self->{stdout} = '';
diff --git a/src/test/recovery/t/037_invalid_database.pl b/src/test/recovery/t/037_invalid_database.pl
index 6d1c711796..e16a3616b2 100644
--- a/src/test/recovery/t/037_invalid_database.pl
+++ b/src/test/recovery/t/037_invalid_database.pl
@@ -96,13 +96,12 @@ my $bgpsql = $node->background_psql('postgres', on_error_stop => 0);
 my $pid = $bgpsql->query('SELECT pg_backend_pid()');
 
 # create the database, prevent drop database via lock held by a 2PC transaction
-ok( $bgpsql->query_safe(
+$bgpsql->query_safe(
 		qq(
   CREATE DATABASE regression_invalid_interrupt;
   BEGIN;
   LOCK pg_tablespace;
-  PREPARE TRANSACTION 'lock_tblspc';)),
-	"blocked DROP DATABASE completion");
+  PREPARE TRANSACTION 'lock_tblspc';));
 
 # Try to drop. This will wait due to the still held lock.
 $bgpsql->query_until(qr//, "DROP DATABASE regression_invalid_interrupt;\n");
@@ -135,11 +134,8 @@ is($node->psql('regression_invalid_interrupt', ''),
 
 # To properly drop the database, we need to release the lock previously preventing
 # doing so.
-ok($bgpsql->query_safe(qq(ROLLBACK PREPARED 'lock_tblspc')),
-	"unblock DROP DATABASE");
-
-ok($bgpsql->query(qq(DROP DATABASE regression_invalid_interrupt)),
-	"DROP DATABASE invalid_interrupt");
+$bgpsql->query_safe(qq(ROLLBACK PREPARED 'lock_tblspc'));
+$bgpsql->query_safe(qq(DROP DATABASE regression_invalid_interrupt));
 
 $bgpsql->quit();
 
-- 
2.34.1

From 18a9531a25ad27eebed2f800346f839f8c8d4e72 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Mon, 8 Jul 2024 10:46:55 -0700
Subject: [PATCH v4 2/4] Test::Cluster: let background_psql() work
 asynchronously

Specifying `wait => 0` as a parameter to background_psql() causes it to
return immediately, which lets the client run code during connection.
(This is useful if, for example, connections are blocked on injected
waitpoints.) Clients later call ->wait_connect() manually to complete
the asynchronous connection.
---
 .../perl/PostgreSQL/Test/BackgroundPsql.pm    | 23 ++++++++++++++-----
 src/test/perl/PostgreSQL/Test/Cluster.pm      | 10 +++++++-
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index 2760e4bc8d..13489ee95e 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -81,7 +81,7 @@ string. For C<interactive> sessions, IO::Pty is required.
 sub new
 {
 	my $class = shift;
-	my ($interactive, $psql_params, $timeout) = @_;
+	my ($interactive, $psql_params, $timeout, $wait) = @_;
 	my $psql = {
 		'stdin' => '',
 		'stdout' => '',
@@ -119,14 +119,25 @@ sub new
 
 	my $self = bless $psql, $class;
 
-	$self->_wait_connect();
+	$wait = 1 unless defined($wait);
+	if ($wait)
+	{
+		$self->wait_connect();
+	}
 
 	return $self;
 }
 
-# Internal routine for awaiting psql starting up and being ready to consume
-# input.
-sub _wait_connect
+=pod
+
+=item $session->wait_connect
+
+Returns once psql has started up and is ready to consume input.  This is called
+automatically for clients unless requested otherwise in the constructor.
+
+=cut
+
+sub wait_connect
 {
 	my ($self) = @_;
 
@@ -187,7 +198,7 @@ sub reconnect_and_clear
 	$self->{stdin} = '';
 	$self->{stdout} = '';
 
-	$self->_wait_connect();
+	$self->wait_connect();
 }
 
 =pod
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 007571e948..aad09cc53e 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2286,6 +2286,12 @@ connection.
 
 If given, it must be an array reference containing additional parameters to B<psql>.
 
+=item wait => 1
+
+By default, this method will not return until connection has completed (or
+failed).  Set B<wait> to 0 to return immediately instead.  (Clients can call the
+session's C<wait_connect> method manually when needed.)
+
 =back
 
 =cut
@@ -2316,13 +2322,15 @@ sub background_psql
 		'-XAtq', '-d', $psql_connstr, '-f', '-');
 
 	$params{on_error_stop} = 1 unless defined $params{on_error_stop};
+	$params{wait} = 1 unless defined $params{wait};
 	$timeout = $params{timeout} if defined $params{timeout};
 
 	push @psql_params, '-v', 'ON_ERROR_STOP=1' if $params{on_error_stop};
 	push @psql_params, @{ $params{extra_params} }
 	  if defined $params{extra_params};
 
-	return PostgreSQL::Test::BackgroundPsql->new(0, \@psql_params, $timeout);
+	return PostgreSQL::Test::BackgroundPsql->new(0, \@psql_params, $timeout,
+		$params{wait});
 }
 
 =pod
-- 
2.34.1

From c8071f91d81906f15eed739877ba75a99028713c Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v4 3/4] pgstat: report in earlier with STATE_STARTING

Add pgstat_bestart_pre_auth(), which reports a 'starting' state while
waiting for backend initialization and client authentication to
complete. Since we hold a transaction open for a good amount of that,
and some authentication methods call out to external systems, having a
pg_stat_activity entry helps DBAs debug when things go badly wrong.
---
 doc/src/sgml/monitoring.sgml                  |  6 ++
 src/backend/utils/activity/backend_status.c   | 37 +++++++++-
 src/backend/utils/adt/pgstatfuncs.c           |  3 +
 src/backend/utils/init/postinit.c             | 20 ++++-
 src/include/utils/backend_status.h            |  2 +
 src/test/authentication/Makefile              |  2 +
 src/test/authentication/meson.build           |  4 +
 .../authentication/t/007_injection_points.pl  | 73 +++++++++++++++++++
 8 files changed, 140 insertions(+), 7 deletions(-)
 create mode 100644 src/test/authentication/t/007_injection_points.pl

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 331315f8d3..81a4a95152 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index bdb3a296ca..d71d7c1b4f 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -69,6 +69,7 @@ static int	localNumBackends = 0;
 static MemoryContext backendStatusSnapContext;
 
 
+static void pgstat_bestart_internal(bool pre_auth);
 static void pgstat_beshutdown_hook(int code, Datum arg);
 static void pgstat_read_current_status(void);
 static void pgstat_setup_backend_status_context(void);
@@ -269,6 +270,34 @@ pgstat_beinit(void)
  */
 void
 pgstat_bestart(void)
+{
+	pgstat_bestart_internal(false);
+}
+
+
+/* ----------
+ * pgstat_bestart_pre_auth() -
+ *
+ *	Like pgstat_beinit(), above, but it's designed to be called before
+ *	authentication has been performed (so we have no user or database IDs).
+ *	Called from InitPostgres.
+ *----------
+ */
+void
+pgstat_bestart_pre_auth(void)
+{
+	pgstat_bestart_internal(true);
+}
+
+
+/* ----------
+ * pgstat_bestart_internal() -
+ *
+ *	Implementation of both flavors of pgstat_bestart().
+ *----------
+ */
+static void
+pgstat_bestart_internal(bool pre_auth)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
@@ -318,9 +347,9 @@ pgstat_bestart(void)
 	lbeentry.st_databaseid = MyDatabaseId;
 
 	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
+	if (!pre_auth && (lbeentry.st_backendType == B_BACKEND
+					  || lbeentry.st_backendType == B_WAL_SENDER
+					  || lbeentry.st_backendType == B_BG_WORKER))
 		lbeentry.st_userid = GetSessionUserId();
 	else
 		lbeentry.st_userid = InvalidOid;
@@ -375,7 +404,7 @@ pgstat_bestart(void)
 	lbeentry.st_gss = false;
 #endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = pre_auth ? STATE_STARTING : STATE_UNDEFINED;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index f7b50e0b5a..c461bbd400 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -366,6 +366,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index a024b1151d..adaa83e745 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -714,6 +715,21 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That
+	 * way, if LWLocks or third-party authentication should happen to hang,
+	 * the DBA will still be able to see what's going on. (A later call to
+	 * pgstat_bestart() will fill in the rest of the status.)
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_pre_auth();
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -782,9 +798,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -882,6 +895,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 97874300c3..8a6d573ce3 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,6 +310,7 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
+extern void pgstat_bestart_pre_auth(void);
 extern void pgstat_bestart(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index da0b71873a..7bbc3b6e57 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,8 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index 8f5688dcc1..09bad8f2b8 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_injection_points.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_injection_points.pl b/src/test/authentication/t/007_injection_points.pl
new file mode 100644
index 0000000000..a6176290a6
--- /dev/null
+++ b/src/test/authentication/t/007_injection_points.pl
@@ -0,0 +1,73 @@
+
+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
+
+# Tests requiring injection_points functionality, to check on behavior that
+# would otherwise race against authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(500_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(500_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
-- 
2.34.1

From d14b97cb7737e32d9d809794a6671b47788879c0 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v4 4/4] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 54 +++++++++++++++----
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 24 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 90 insertions(+), 11 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 81a4a95152..a148e63711 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 47e8c91606..bbcde591ae 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -1000,6 +1001,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1011,6 +1013,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1222,6 +1225,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1231,6 +1235,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1296,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1305,6 +1312,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1402,19 +1410,25 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
 	if (!port->hba->compat_realm)
 	{
-		int			status = pg_SSPI_make_upn(accountname, sizeof(accountname),
-											  domainname, sizeof(domainname),
-											  port->hba->upn_username);
+		int			status;
+
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_MAKE_UPN);
+		status = pg_SSPI_make_upn(accountname, sizeof(accountname),
+								  domainname, sizeof(domainname),
+								  port->hba->upn_username);
+		pgstat_report_wait_end();
 
 		if (status != STATUS_OK)
 			/* Error already reported from pg_SSPI_make_upn */
@@ -2119,7 +2133,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2132,7 +2148,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2483,7 +2501,11 @@ CheckLDAPAuth(Port *port)
 	if (passwd == NULL)
 		return STATUS_EOF;		/* client wouldn't send password */
 
-	if (InitializeLDAPConnection(port, &ldap) == STATUS_ERROR)
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_INITIALIZE);
+	r = InitializeLDAPConnection(port, &ldap);
+	pgstat_report_wait_end();
+
+	if (r == STATUS_ERROR)
 	{
 		/* Error message already sent */
 		pfree(passwd);
@@ -2530,9 +2552,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2555,6 +2580,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2562,6 +2589,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2630,7 +2658,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -2890,12 +2920,16 @@ CheckRADIUSAuth(Port *port)
 	identifiers = list_head(port->hba->radiusidentifiers);
 	foreach(server, port->hba->radiusservers)
 	{
-		int			ret = PerformRadiusTransaction(lfirst(server),
-												   lfirst(secrets),
-												   radiusports ? lfirst(radiusports) : NULL,
-												   identifiers ? lfirst(identifiers) : NULL,
-												   port->user_name,
-												   passwd);
+		int			ret;
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_TRANSACTION);
+		ret = PerformRadiusTransaction(lfirst(server),
+									   lfirst(secrets),
+									   radiusports ? lfirst(radiusports) : NULL,
+									   identifiers ? lfirst(identifiers) : NULL,
+									   port->user_name,
+									   passwd);
+		pgstat_report_wait_end();
 
 		/*------
 		 * STATUS_OK = Login OK
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d930277140..a388999b1a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index 8efb4044d6..e5c27a2d2c 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -162,6 +162,30 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_INITIALIZE	"Waiting to initialize an LDAP connection."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+PAM_ACCT_MGMT	"Waiting for the local PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the local PAM service to authenticate the user."
+RADIUS_TRANSACTION	"Waiting for a RADIUS transaction to complete."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's account SID."
+SSPI_MAKE_UPN	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 9f18a753d4..014d536441 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index fad7fc3a7e..030d1a8321 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -179,6 +179,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -187,7 +188,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

1:  64289b97e5 < -:  ---------- BackgroundPsql: handle empty query results
2:  18a9531a25 < -:  ---------- Test::Cluster: let background_psql() work asynchronously
3:  c8071f91d8 ! 1:  e755fdccf1 pgstat: report in earlier with STATE_STARTING
    @@ src/test/authentication/Makefile: subdir = src/test/authentication
      top_builddir = ../../..
      include $(top_builddir)/src/Makefile.global
      
    ++EXTRA_INSTALL = src/test/modules/injection_points
    ++
     +export enable_injection_points
     +
      check:
    @@ src/test/authentication/meson.build: tests += {
            't/004_file_inclusion.pl',
            't/005_sspi.pl',
            't/006_login_trigger.pl',
    -+      't/007_injection_points.pl',
    ++      't/007_pre_auth.pl',
          ],
        },
      }
     
    - ## src/test/authentication/t/007_injection_points.pl (new) ##
    + ## src/test/authentication/t/007_pre_auth.pl (new) ##
     @@
     +
     +# Copyright (c) 2021-2024, PostgreSQL Global Development Group
     +
    -+# Tests requiring injection_points functionality, to check on behavior that
    -+# would otherwise race against authentication.
    ++# Tests for connection behavior prior to authentication.
     +
     +use strict;
     +use warnings FATAL => 'all';
    @@ src/test/authentication/t/007_injection_points.pl (new)
     +]);
     +
     +$node->start;
    ++
    ++# Check if the extension injection_points is available, as it may be
    ++# possible that this script is run with installcheck, where the module
    ++# would not be installed by default.
    ++if (!$node->check_extension('injection_points'))
    ++{
    ++	plan skip_all => 'Extension injection_points not installed';
    ++}
    ++
     +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
     +
     +# Connect to the server and inject a waitpoint.
4:  d14b97cb77 ! 2:  858e95f996 Report external auth calls as wait events
    @@ src/backend/utils/activity/wait_event_names.txt: XACT_GROUP_UPDATE	"Waiting for
     +LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
     +LDAP_INITIALIZE	"Waiting to initialize an LDAP connection."
     +LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
    -+PAM_ACCT_MGMT	"Waiting for the local PAM service to validate the user account."
    -+PAM_AUTHENTICATE	"Waiting for the local PAM service to authenticate the user."
    ++PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
    ++PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
     +RADIUS_TRANSACTION	"Waiting for a RADIUS transaction to complete."
     +SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
     +SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
    -+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's account SID."
    ++SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
     +SSPI_MAKE_UPN	"Waiting for Windows to translate a Kerberos UPN."
     +
     +ABI_compatibility:

From e755fdccf16cb4fcd3036e1209291750ffecd261 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v5 1/2] pgstat: report in earlier with STATE_STARTING

Add pgstat_bestart_pre_auth(), which reports a 'starting' state while
waiting for backend initialization and client authentication to
complete. Since we hold a transaction open for a good amount of that,
and some authentication methods call out to external systems, having a
pg_stat_activity entry helps DBAs debug when things go badly wrong.
---
 doc/src/sgml/monitoring.sgml                |  6 ++
 src/backend/utils/activity/backend_status.c | 37 +++++++++-
 src/backend/utils/adt/pgstatfuncs.c         |  3 +
 src/backend/utils/init/postinit.c           | 20 ++++-
 src/include/utils/backend_status.h          |  2 +
 src/test/authentication/Makefile            |  4 +
 src/test/authentication/meson.build         |  4 +
 src/test/authentication/t/007_pre_auth.pl   | 81 +++++++++++++++++++++
 8 files changed, 150 insertions(+), 7 deletions(-)
 create mode 100644 src/test/authentication/t/007_pre_auth.pl

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 331315f8d3..81a4a95152 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index bdb3a296ca..d71d7c1b4f 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -69,6 +69,7 @@ static int	localNumBackends = 0;
 static MemoryContext backendStatusSnapContext;
 
 
+static void pgstat_bestart_internal(bool pre_auth);
 static void pgstat_beshutdown_hook(int code, Datum arg);
 static void pgstat_read_current_status(void);
 static void pgstat_setup_backend_status_context(void);
@@ -269,6 +270,34 @@ pgstat_beinit(void)
  */
 void
 pgstat_bestart(void)
+{
+	pgstat_bestart_internal(false);
+}
+
+
+/* ----------
+ * pgstat_bestart_pre_auth() -
+ *
+ *	Like pgstat_beinit(), above, but it's designed to be called before
+ *	authentication has been performed (so we have no user or database IDs).
+ *	Called from InitPostgres.
+ *----------
+ */
+void
+pgstat_bestart_pre_auth(void)
+{
+	pgstat_bestart_internal(true);
+}
+
+
+/* ----------
+ * pgstat_bestart_internal() -
+ *
+ *	Implementation of both flavors of pgstat_bestart().
+ *----------
+ */
+static void
+pgstat_bestart_internal(bool pre_auth)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
@@ -318,9 +347,9 @@ pgstat_bestart(void)
 	lbeentry.st_databaseid = MyDatabaseId;
 
 	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
+	if (!pre_auth && (lbeentry.st_backendType == B_BACKEND
+					  || lbeentry.st_backendType == B_WAL_SENDER
+					  || lbeentry.st_backendType == B_BG_WORKER))
 		lbeentry.st_userid = GetSessionUserId();
 	else
 		lbeentry.st_userid = InvalidOid;
@@ -375,7 +404,7 @@ pgstat_bestart(void)
 	lbeentry.st_gss = false;
 #endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = pre_auth ? STATE_STARTING : STATE_UNDEFINED;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index f7b50e0b5a..c461bbd400 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -366,6 +366,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index a024b1151d..adaa83e745 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -714,6 +715,21 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That
+	 * way, if LWLocks or third-party authentication should happen to hang,
+	 * the DBA will still be able to see what's going on. (A later call to
+	 * pgstat_bestart() will fill in the rest of the status.)
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_pre_auth();
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -782,9 +798,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -882,6 +895,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 97874300c3..8a6d573ce3 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,6 +310,7 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
+extern void pgstat_bestart_pre_auth(void);
 extern void pgstat_bestart(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index da0b71873a..d05b15de7c 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,10 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+EXTRA_INSTALL = src/test/modules/injection_points
+
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index 8f5688dcc1..fe4ca51873 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_pre_auth.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
new file mode 100644
index 0000000000..dd554462d3
--- /dev/null
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -0,0 +1,81 @@
+
+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
+
+# Tests for connection behavior prior to authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(500_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(500_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
-- 
2.34.1

From 858e95f996589461e2840047fa35675b6f96e46d Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v5 2/2] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 54 +++++++++++++++----
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 24 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 90 insertions(+), 11 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 81a4a95152..a148e63711 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 47e8c91606..bbcde591ae 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -1000,6 +1001,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1011,6 +1013,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1222,6 +1225,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1231,6 +1235,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1296,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1305,6 +1312,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1402,19 +1410,25 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
 	if (!port->hba->compat_realm)
 	{
-		int			status = pg_SSPI_make_upn(accountname, sizeof(accountname),
-											  domainname, sizeof(domainname),
-											  port->hba->upn_username);
+		int			status;
+
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_MAKE_UPN);
+		status = pg_SSPI_make_upn(accountname, sizeof(accountname),
+								  domainname, sizeof(domainname),
+								  port->hba->upn_username);
+		pgstat_report_wait_end();
 
 		if (status != STATUS_OK)
 			/* Error already reported from pg_SSPI_make_upn */
@@ -2119,7 +2133,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2132,7 +2148,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2483,7 +2501,11 @@ CheckLDAPAuth(Port *port)
 	if (passwd == NULL)
 		return STATUS_EOF;		/* client wouldn't send password */
 
-	if (InitializeLDAPConnection(port, &ldap) == STATUS_ERROR)
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_INITIALIZE);
+	r = InitializeLDAPConnection(port, &ldap);
+	pgstat_report_wait_end();
+
+	if (r == STATUS_ERROR)
 	{
 		/* Error message already sent */
 		pfree(passwd);
@@ -2530,9 +2552,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2555,6 +2580,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2562,6 +2589,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2630,7 +2658,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -2890,12 +2920,16 @@ CheckRADIUSAuth(Port *port)
 	identifiers = list_head(port->hba->radiusidentifiers);
 	foreach(server, port->hba->radiusservers)
 	{
-		int			ret = PerformRadiusTransaction(lfirst(server),
-												   lfirst(secrets),
-												   radiusports ? lfirst(radiusports) : NULL,
-												   identifiers ? lfirst(identifiers) : NULL,
-												   port->user_name,
-												   passwd);
+		int			ret;
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_TRANSACTION);
+		ret = PerformRadiusTransaction(lfirst(server),
+									   lfirst(secrets),
+									   radiusports ? lfirst(radiusports) : NULL,
+									   identifiers ? lfirst(identifiers) : NULL,
+									   port->user_name,
+									   passwd);
+		pgstat_report_wait_end();
 
 		/*------
 		 * STATUS_OK = Login OK
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d930277140..a388999b1a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index 16144c2b72..a341c6e7c5 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -161,6 +161,30 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_INITIALIZE	"Waiting to initialize an LDAP connection."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_TRANSACTION	"Waiting for a RADIUS transaction to complete."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_MAKE_UPN	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 9f18a753d4..014d536441 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index fad7fc3a7e..030d1a8321 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -179,6 +179,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -187,7 +188,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

1:  e755fdccf1 = 1:  e755fdccf1 pgstat: report in earlier with STATE_STARTING
2:  858e95f996 ! 2:  3f14df0308 Report external auth calls as wait events
    @@ src/backend/libpq/auth.c: pg_SSPI_recvauth(Port *port)
      
      	free(tokenuser);
      
    - 	if (!port->hba->compat_realm)
    - 	{
    --		int			status = pg_SSPI_make_upn(accountname, sizeof(accountname),
    --											  domainname, sizeof(domainname),
    --											  port->hba->upn_username);
    -+		int			status;
    +@@ src/backend/libpq/auth.c: pg_SSPI_make_upn(char *accountname,
    + 	 */
    + 
    + 	samname = psprintf("%s\\%s", domainname, accountname);
     +
    -+		pgstat_report_wait_start(WAIT_EVENT_SSPI_MAKE_UPN);
    -+		status = pg_SSPI_make_upn(accountname, sizeof(accountname),
    -+								  domainname, sizeof(domainname),
    -+								  port->hba->upn_username);
    -+		pgstat_report_wait_end();
    ++	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
    + 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
    + 						NULL, &upnamesize);
    ++	pgstat_report_wait_end();
    + 
    + 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
    + 		|| upnamesize == 0)
    +@@ src/backend/libpq/auth.c: pg_SSPI_make_upn(char *accountname,
    + 	/* upnamesize includes the terminating NUL. */
    + 	upname = palloc(upnamesize);
    + 
    ++	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
    + 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
    + 						upname, &upnamesize);
    ++	pgstat_report_wait_end();
      
    - 		if (status != STATUS_OK)
    - 			/* Error already reported from pg_SSPI_make_upn */
    + 	pfree(samname);
    + 	if (res)
     @@ src/backend/libpq/auth.c: CheckPAMAuth(Port *port, const char *user, const char *password)
      		return STATUS_ERROR;
      	}
    @@ src/backend/libpq/auth.c: CheckPAMAuth(Port *port, const char *user, const char
      
      	if (retval != PAM_SUCCESS)
      	{
    -@@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
    - 	if (passwd == NULL)
    - 		return STATUS_EOF;		/* client wouldn't send password */
    +@@ src/backend/libpq/auth.c: InitializeLDAPConnection(Port *port, LDAP **ldap)
    + 			}
      
    --	if (InitializeLDAPConnection(port, &ldap) == STATUS_ERROR)
    -+	pgstat_report_wait_start(WAIT_EVENT_LDAP_INITIALIZE);
    -+	r = InitializeLDAPConnection(port, &ldap);
    -+	pgstat_report_wait_end();
    + 			/* Look up a list of LDAP server hosts and port numbers */
    +-			if (ldap_domain2hostlist(domain, &hostlist))
    ++			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
    ++			r = ldap_domain2hostlist(domain, &hostlist);
    ++			pgstat_report_wait_end();
     +
    -+	if (r == STATUS_ERROR)
    ++			if (r)
    + 			{
    + 				ereport(LOG,
    + 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
    +@@ src/backend/libpq/auth.c: InitializeLDAPConnection(Port *port, LDAP **ldap)
    + 
    + 	if (port->hba->ldaptls)
      	{
    - 		/* Error message already sent */
    - 		pfree(passwd);
    ++		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
    + #ifndef WIN32
    +-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
    ++		r = ldap_start_tls_s(*ldap, NULL, NULL);
    + #else
    +-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
    ++		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
    + #endif
    ++		pgstat_report_wait_end();
    ++
    ++		if (r != LDAP_SUCCESS)
    + 		{
    + 			ereport(LOG,
    + 					(errmsg("could not start LDAP TLS session: %s",
     @@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
      		 * Bind with a pre-defined username/password (if available) for
      		 * searching. If none is specified, this turns into an anonymous bind.
    @@ src/backend/libpq/auth.c: CheckLDAPAuth(Port *port)
      
      	if (r != LDAP_SUCCESS)
      	{
    -@@ src/backend/libpq/auth.c: CheckRADIUSAuth(Port *port)
    - 	identifiers = list_head(port->hba->radiusidentifiers);
    - 	foreach(server, port->hba->radiusservers)
    - 	{
    --		int			ret = PerformRadiusTransaction(lfirst(server),
    --												   lfirst(secrets),
    --												   radiusports ? lfirst(radiusports) : NULL,
    --												   identifiers ? lfirst(identifiers) : NULL,
    --												   port->user_name,
    --												   passwd);
    -+		int			ret;
    +@@ src/backend/libpq/auth.c: PerformRadiusTransaction(const char *server, const char *secret, const char *por
    + 		return STATUS_ERROR;
    + 	}
    + 
    +-	if (sendto(sock, radius_buffer, packetlength, 0,
    +-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
    ++	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
    ++	r = sendto(sock, radius_buffer, packetlength, 0,
    ++			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
    ++	pgstat_report_wait_end();
     +
    -+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_TRANSACTION);
    -+		ret = PerformRadiusTransaction(lfirst(server),
    -+									   lfirst(secrets),
    -+									   radiusports ? lfirst(radiusports) : NULL,
    -+									   identifiers ? lfirst(identifiers) : NULL,
    -+									   port->user_name,
    -+									   passwd);
    ++	if (r < 0)
    + 	{
    + 		ereport(LOG,
    + 				(errmsg("could not send RADIUS packet: %m")));
    +@@ src/backend/libpq/auth.c: PerformRadiusTransaction(const char *server, const char *secret, const char *por
    + 		FD_ZERO(&fdset);
    + 		FD_SET(sock, &fdset);
    + 
    ++		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
    + 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
     +		pgstat_report_wait_end();
    ++
    + 		if (r < 0)
    + 		{
    + 			if (errno == EINTR)
    +@@ src/backend/libpq/auth.c: PerformRadiusTransaction(const char *server, const char *secret, const char *por
    + 		 */
      
    - 		/*------
    - 		 * STATUS_OK = Login OK
    + 		addrsize = sizeof(remoteaddr);
    ++
    ++		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
    + 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
    + 								(struct sockaddr *) &remoteaddr, &addrsize);
    ++		pgstat_report_wait_end();
    ++
    + 		if (packetlength < 0)
    + 		{
    + 			ereport(LOG,
     
      ## src/backend/utils/activity/wait_event.c ##
     @@ src/backend/utils/activity/wait_event.c: static const char *pgstat_get_wait_client(WaitEventClient w);
    @@ src/backend/utils/activity/wait_event_names.txt: XACT_GROUP_UPDATE	"Waiting for
     +GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
     +LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
     +LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
    -+LDAP_INITIALIZE	"Waiting to initialize an LDAP connection."
    ++LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
     +LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
    ++LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
     +PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
     +PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
    -+RADIUS_TRANSACTION	"Waiting for a RADIUS transaction to complete."
    ++RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
    ++RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
    ++RADIUS_WAIT	"Waiting for a RADIUS server to respond."
     +SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
     +SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
     +SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
    -+SSPI_MAKE_UPN	"Waiting for Windows to translate a Kerberos UPN."
    ++SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
     +
     +ABI_compatibility:
     +
-:  ---------- > 3:  fb21328568 squash! Report external auth calls as wait events

From e755fdccf16cb4fcd3036e1209291750ffecd261 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v6 1/3] pgstat: report in earlier with STATE_STARTING

Add pgstat_bestart_pre_auth(), which reports a 'starting' state while
waiting for backend initialization and client authentication to
complete. Since we hold a transaction open for a good amount of that,
and some authentication methods call out to external systems, having a
pg_stat_activity entry helps DBAs debug when things go badly wrong.
---
 doc/src/sgml/monitoring.sgml                |  6 ++
 src/backend/utils/activity/backend_status.c | 37 +++++++++-
 src/backend/utils/adt/pgstatfuncs.c         |  3 +
 src/backend/utils/init/postinit.c           | 20 ++++-
 src/include/utils/backend_status.h          |  2 +
 src/test/authentication/Makefile            |  4 +
 src/test/authentication/meson.build         |  4 +
 src/test/authentication/t/007_pre_auth.pl   | 81 +++++++++++++++++++++
 8 files changed, 150 insertions(+), 7 deletions(-)
 create mode 100644 src/test/authentication/t/007_pre_auth.pl

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 331315f8d3..81a4a95152 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index bdb3a296ca..d71d7c1b4f 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -69,6 +69,7 @@ static int	localNumBackends = 0;
 static MemoryContext backendStatusSnapContext;
 
 
+static void pgstat_bestart_internal(bool pre_auth);
 static void pgstat_beshutdown_hook(int code, Datum arg);
 static void pgstat_read_current_status(void);
 static void pgstat_setup_backend_status_context(void);
@@ -269,6 +270,34 @@ pgstat_beinit(void)
  */
 void
 pgstat_bestart(void)
+{
+	pgstat_bestart_internal(false);
+}
+
+
+/* ----------
+ * pgstat_bestart_pre_auth() -
+ *
+ *	Like pgstat_beinit(), above, but it's designed to be called before
+ *	authentication has been performed (so we have no user or database IDs).
+ *	Called from InitPostgres.
+ *----------
+ */
+void
+pgstat_bestart_pre_auth(void)
+{
+	pgstat_bestart_internal(true);
+}
+
+
+/* ----------
+ * pgstat_bestart_internal() -
+ *
+ *	Implementation of both flavors of pgstat_bestart().
+ *----------
+ */
+static void
+pgstat_bestart_internal(bool pre_auth)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
@@ -318,9 +347,9 @@ pgstat_bestart(void)
 	lbeentry.st_databaseid = MyDatabaseId;
 
 	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
+	if (!pre_auth && (lbeentry.st_backendType == B_BACKEND
+					  || lbeentry.st_backendType == B_WAL_SENDER
+					  || lbeentry.st_backendType == B_BG_WORKER))
 		lbeentry.st_userid = GetSessionUserId();
 	else
 		lbeentry.st_userid = InvalidOid;
@@ -375,7 +404,7 @@ pgstat_bestart(void)
 	lbeentry.st_gss = false;
 #endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = pre_auth ? STATE_STARTING : STATE_UNDEFINED;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index f7b50e0b5a..c461bbd400 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -366,6 +366,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index a024b1151d..adaa83e745 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -714,6 +715,21 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That
+	 * way, if LWLocks or third-party authentication should happen to hang,
+	 * the DBA will still be able to see what's going on. (A later call to
+	 * pgstat_bestart() will fill in the rest of the status.)
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_pre_auth();
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -782,9 +798,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -882,6 +895,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 97874300c3..8a6d573ce3 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,6 +310,7 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
+extern void pgstat_bestart_pre_auth(void);
 extern void pgstat_bestart(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index da0b71873a..d05b15de7c 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,10 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+EXTRA_INSTALL = src/test/modules/injection_points
+
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index 8f5688dcc1..fe4ca51873 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_pre_auth.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
new file mode 100644
index 0000000000..dd554462d3
--- /dev/null
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -0,0 +1,81 @@
+
+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
+
+# Tests for connection behavior prior to authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(500_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(500_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
-- 
2.34.1

From 3f14df0308d9bbea9ab81a3c7021bc82f3c50db6 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v6 2/3] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 56 +++++++++++++++++--
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 27 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 81a4a95152..a148e63711 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 47e8c91606..fe4cbeb8a0 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -1000,6 +1001,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1011,6 +1013,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1222,6 +1225,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1231,6 +1235,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1296,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1305,6 +1312,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1402,11 +1410,13 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
@@ -1503,8 +1513,11 @@ pg_SSPI_make_upn(char *accountname,
 	 */
 
 	samname = psprintf("%s\\%s", domainname, accountname);
+
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						NULL, &upnamesize);
+	pgstat_report_wait_end();
 
 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
 		|| upnamesize == 0)
@@ -1519,8 +1532,10 @@ pg_SSPI_make_upn(char *accountname,
 	/* upnamesize includes the terminating NUL. */
 	upname = palloc(upnamesize);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						upname, &upnamesize);
+	pgstat_report_wait_end();
 
 	pfree(samname);
 	if (res)
@@ -2119,7 +2134,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2132,7 +2149,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2272,7 +2291,11 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 			}
 
 			/* Look up a list of LDAP server hosts and port numbers */
-			if (ldap_domain2hostlist(domain, &hostlist))
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
+			r = ldap_domain2hostlist(domain, &hostlist);
+			pgstat_report_wait_end();
+
+			if (r)
 			{
 				ereport(LOG,
 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
@@ -2366,11 +2389,15 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 
 	if (port->hba->ldaptls)
 	{
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
 #ifndef WIN32
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL);
 #else
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
 #endif
+		pgstat_report_wait_end();
+
+		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
 					(errmsg("could not start LDAP TLS session: %s",
@@ -2530,9 +2557,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2555,6 +2585,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2562,6 +2594,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2630,7 +2663,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -3079,8 +3114,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		return STATUS_ERROR;
 	}
 
-	if (sendto(sock, radius_buffer, packetlength, 0,
-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
+	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
+	r = sendto(sock, radius_buffer, packetlength, 0,
+			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
+	pgstat_report_wait_end();
+
+	if (r < 0)
 	{
 		ereport(LOG,
 				(errmsg("could not send RADIUS packet: %m")));
@@ -3128,7 +3167,10 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		FD_ZERO(&fdset);
 		FD_SET(sock, &fdset);
 
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
+		pgstat_report_wait_end();
+
 		if (r < 0)
 		{
 			if (errno == EINTR)
@@ -3161,8 +3203,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 
 		addrsize = sizeof(remoteaddr);
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
 								(struct sockaddr *) &remoteaddr, &addrsize);
+		pgstat_report_wait_end();
+
 		if (packetlength < 0)
 		{
 			ereport(LOG,
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d930277140..a388999b1a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index 16144c2b72..db8474bc85 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -161,6 +161,33 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
+RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
+RADIUS_WAIT	"Waiting for a RADIUS server to respond."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 9f18a753d4..014d536441 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index fad7fc3a7e..030d1a8321 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -179,6 +179,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -187,7 +188,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

From fb21328568d33ce57f6bc79c4944cea79ea9fce7 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 8 Nov 2024 14:19:26 -0800
Subject: [PATCH v6 3/3] squash! Report external auth calls as wait events

Add a wait event around all calls to ldap_unbind().
---
 src/backend/libpq/auth.c                      | 33 ++++++++++++++-----
 .../utils/activity/wait_event_names.txt       |  1 +
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index fe4cbeb8a0..df38d66f48 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -2226,6 +2226,7 @@ CheckBSDAuth(Port *port, char *user)
 #ifdef USE_LDAP
 
 static int	errdetail_for_ldap(LDAP *ldap);
+static void unbind(LDAP *ldap);
 
 /*
  * Initialize a connection to the LDAP server, including setting up
@@ -2383,7 +2384,7 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 				(errmsg("could not set LDAP protocol version: %s",
 						ldap_err2string(r)),
 				 errdetail_for_ldap(*ldap)));
-		ldap_unbind(*ldap);
+		unbind(*ldap);
 		return STATUS_ERROR;
 	}
 
@@ -2403,7 +2404,7 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 					(errmsg("could not start LDAP TLS session: %s",
 							ldap_err2string(r)),
 					 errdetail_for_ldap(*ldap)));
-			ldap_unbind(*ldap);
+			unbind(*ldap);
 			return STATUS_ERROR;
 		}
 	}
@@ -2547,7 +2548,7 @@ CheckLDAPAuth(Port *port)
 			{
 				ereport(LOG,
 						(errmsg("invalid character in user name for LDAP authentication")));
-				ldap_unbind(ldap);
+				unbind(ldap);
 				pfree(passwd);
 				return STATUS_ERROR;
 			}
@@ -2571,7 +2572,7 @@ CheckLDAPAuth(Port *port)
 							server_name,
 							ldap_err2string(r)),
 					 errdetail_for_ldap(ldap)));
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			return STATUS_ERROR;
 		}
@@ -2604,7 +2605,7 @@ CheckLDAPAuth(Port *port)
 					 errdetail_for_ldap(ldap)));
 			if (search_message != NULL)
 				ldap_msgfree(search_message);
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			pfree(filter);
 			return STATUS_ERROR;
@@ -2626,7 +2627,7 @@ CheckLDAPAuth(Port *port)
 										  count,
 										  filter, server_name, count)));
 
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2645,7 +2646,7 @@ CheckLDAPAuth(Port *port)
 							filter, server_name,
 							ldap_err2string(error)),
 					 errdetail_for_ldap(ldap)));
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2673,7 +2674,7 @@ CheckLDAPAuth(Port *port)
 				(errmsg("LDAP login failed for user \"%s\" on server \"%s\": %s",
 						fulluser, server_name, ldap_err2string(r)),
 				 errdetail_for_ldap(ldap)));
-		ldap_unbind(ldap);
+		unbind(ldap);
 		pfree(passwd);
 		pfree(fulluser);
 		return STATUS_ERROR;
@@ -2682,7 +2683,7 @@ CheckLDAPAuth(Port *port)
 	/* Save the original bind DN as the authenticated identity. */
 	set_authn_id(port, fulluser);
 
-	ldap_unbind(ldap);
+	unbind(ldap);
 	pfree(passwd);
 	pfree(fulluser);
 
@@ -2709,6 +2710,20 @@ errdetail_for_ldap(LDAP *ldap)
 	return 0;
 }
 
+/*
+ * ldap_unbind() is not a lightweight operation; it writes data to the socket
+ * and may need to tear down (LDAP) SASL mechanisms, depending on the LDAP
+ * implementation in use. This function just wraps the unbind with pgstat
+ * reporting in case something takes longer than expected.
+ */
+static void
+unbind(LDAP *ldap)
+{
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND);
+	ldap_unbind(ldap);
+	pgstat_report_wait_end();
+}
+
 #endif							/* USE_LDAP */
 
 
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index db8474bc85..00a9459ad4 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -176,6 +176,7 @@ LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory
 LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
 LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
 LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+LDAP_UNBIND	"Waiting for an LDAP connection to be unbound and closed."
 PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
 PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
 RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
-- 
2.34.1

From b91a602cab8ee13a61168e1967648249e28a22e3 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v7 1/3] pgstat: report in earlier with STATE_STARTING

Split pgstat_bestart() into three phases for better observability:

1) pgstat_bestart_initial() reports a 'starting' state while waiting for
   backend initialization and client authentication to complete.  Since
   we hold a transaction open for a good amount of that, and some
   authentication methods call out to external systems, having an early
   pg_stat_activity entry helps DBAs debug when things go badly wrong.

2) pgstat_bestart_security() reports the SSL/GSS status of the
   connection.  Some backends don't call this at all; others call it
   twice, once after transport establishment and once after client
   authentication.

3) pgstat_bestart_final() reports the user and database IDs, takes the
   entry out of STATE_STARTING, and reports the application_name.
   TODO: should the order of those last two be swapped?
---
 doc/src/sgml/monitoring.sgml                |   6 +
 src/backend/postmaster/auxprocess.c         |   3 +-
 src/backend/utils/activity/backend_status.c | 207 +++++++++++++-------
 src/backend/utils/adt/pgstatfuncs.c         |   3 +
 src/backend/utils/init/postinit.c           |  42 +++-
 src/include/utils/backend_status.h          |   5 +-
 src/test/authentication/Makefile            |   4 +
 src/test/authentication/meson.build         |   4 +
 src/test/authentication/t/007_pre_auth.pl   |  81 ++++++++
 src/test/ssl/Makefile                       |   3 +-
 src/test/ssl/meson.build                    |   1 +
 src/test/ssl/t/001_ssltests.pl              |  65 ++++++
 12 files changed, 335 insertions(+), 89 deletions(-)
 create mode 100644 src/test/authentication/t/007_pre_auth.pl

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 331315f8d3..81a4a95152 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
diff --git a/src/backend/postmaster/auxprocess.c b/src/backend/postmaster/auxprocess.c
index d19174bda3..bcfd231d0d 100644
--- a/src/backend/postmaster/auxprocess.c
+++ b/src/backend/postmaster/auxprocess.c
@@ -78,7 +78,8 @@ AuxiliaryProcessMainCommon(void)
 
 	/* Initialize backend status information */
 	pgstat_beinit();
-	pgstat_bestart();
+	pgstat_bestart_initial();
+	pgstat_bestart_final();
 
 	/* register a before-shutdown callback for LWLock cleanup */
 	before_shmem_exit(ShutdownAuxiliaryProcess, 0);
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index bdb3a296ca..d8f87cbc92 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -253,31 +253,18 @@ pgstat_beinit(void)
 	on_shmem_exit(pgstat_beshutdown_hook, 0);
 }
 
-
-/* ----------
- * pgstat_bestart() -
- *
- *	Initialize this backend's entry in the PgBackendStatus array.
- *	Called from InitPostgres.
- *
- *	Apart from auxiliary processes, MyDatabaseId, session userid, and
- *	application_name must already be set (hence, this cannot be combined
- *	with pgstat_beinit).  Note also that we must be inside a transaction
- *	if this isn't an aux process, as we may need to do encoding conversion
- *	on some strings.
- *----------
+/*
+ * Clears out a new pgstat entry, initializing it to suitable defaults and
+ * reporting STATE_STARTING. Backends should continue filling in any transport
+ * security details as needed with pgstat_bestart_security(), and must finally
+ * exit STATE_STARTING by calling pgstat_bestart_final(), once user and database
+ * IDs have been determined.
  */
 void
-pgstat_bestart(void)
+pgstat_bestart_initial(void)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
-#ifdef USE_SSL
-	PgBackendSSLStatus lsslstatus;
-#endif
-#ifdef ENABLE_GSS
-	PgBackendGSSStatus lgssstatus;
-#endif
 
 	/* pgstats state must be initialized from pgstat_beinit() */
 	Assert(vbeentry != NULL);
@@ -297,14 +284,6 @@ pgstat_bestart(void)
 		   unvolatize(PgBackendStatus *, vbeentry),
 		   sizeof(PgBackendStatus));
 
-	/* These structs can just start from zeroes each time, though */
-#ifdef USE_SSL
-	memset(&lsslstatus, 0, sizeof(lsslstatus));
-#endif
-#ifdef ENABLE_GSS
-	memset(&lgssstatus, 0, sizeof(lgssstatus));
-#endif
-
 	/*
 	 * Now fill in all the fields of lbeentry, except for strings that are
 	 * out-of-line data.  Those have to be handled separately, below.
@@ -315,15 +294,8 @@ pgstat_bestart(void)
 	lbeentry.st_activity_start_timestamp = 0;
 	lbeentry.st_state_start_timestamp = 0;
 	lbeentry.st_xact_start_timestamp = 0;
-	lbeentry.st_databaseid = MyDatabaseId;
-
-	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
-		lbeentry.st_userid = GetSessionUserId();
-	else
-		lbeentry.st_userid = InvalidOid;
+	lbeentry.st_databaseid = InvalidOid;
+	lbeentry.st_userid = InvalidOid;
 
 	/*
 	 * We may not have a MyProcPort (eg, if this is the autovacuum process).
@@ -336,46 +308,10 @@ pgstat_bestart(void)
 	else
 		MemSet(&lbeentry.st_clientaddr, 0, sizeof(lbeentry.st_clientaddr));
 
-#ifdef USE_SSL
-	if (MyProcPort && MyProcPort->ssl_in_use)
-	{
-		lbeentry.st_ssl = true;
-		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
-		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
-		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
-		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
-		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
-		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_ssl = false;
-	}
-#else
 	lbeentry.st_ssl = false;
-#endif
-
-#ifdef ENABLE_GSS
-	if (MyProcPort && MyProcPort->gss != NULL)
-	{
-		const char *princ = be_gssapi_get_princ(MyProcPort);
-
-		lbeentry.st_gss = true;
-		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
-		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
-		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
-		if (princ)
-			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_gss = false;
-	}
-#else
 	lbeentry.st_gss = false;
-#endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = STATE_STARTING;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
@@ -417,16 +353,135 @@ pgstat_bestart(void)
 	lbeentry.st_clienthostname[NAMEDATALEN - 1] = '\0';
 	lbeentry.st_activity_raw[pgstat_track_activity_query_size - 1] = '\0';
 
+	/* These structs can just start from zeroes each time */
 #ifdef USE_SSL
-	memcpy(lbeentry.st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+	memset(lbeentry.st_sslstatus, 0, sizeof(PgBackendSSLStatus));
 #endif
 #ifdef ENABLE_GSS
-	memcpy(lbeentry.st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+	memset(lbeentry.st_gssstatus, 0, sizeof(PgBackendGSSStatus));
 #endif
 
 	PGSTAT_END_WRITE_ACTIVITY(vbeentry);
+}
+
+/*
+ * Fill in SSL and GSS information for the pgstat entry. This is separate from
+ * pgstat_bestart_initial() so that backends may call it multiple times as
+ * security details are filled in.
+ *
+ * This should only be called from backends with a MyProcPort.
+ */
+void
+pgstat_bestart_security(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	bool		ssl = false;
+	bool		gss = false;
+#ifdef USE_SSL
+	PgBackendSSLStatus lsslstatus;
+	PgBackendSSLStatus *st_sslstatus;
+#endif
+#ifdef ENABLE_GSS
+	PgBackendGSSStatus lgssstatus;
+	PgBackendGSSStatus *st_gssstatus;
+#endif
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+	Assert(MyProcPort);			/* otherwise there's no point */
+
+#ifdef USE_SSL
+	st_sslstatus = beentry->st_sslstatus;
+	memset(&lsslstatus, 0, sizeof(lsslstatus));
+
+	if (MyProcPort->ssl_in_use)
+	{
+		ssl = true;
+		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
+		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
+		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
+		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
+		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
+		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
+	}
+#endif
+
+#ifdef ENABLE_GSS
+	st_gssstatus = beentry->st_gssstatus;
+	memset(&lgssstatus, 0, sizeof(lgssstatus));
+
+	if (MyProcPort->gss != NULL)
+	{
+		const char *princ = be_gssapi_get_princ(MyProcPort);
+
+		gss = true;
+		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
+		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
+		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
+		if (princ)
+			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
+	}
+#endif
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_ssl = ssl;
+	beentry->st_gss = gss;
+
+#ifdef USE_SSL
+	memcpy(st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+#endif
+#ifdef ENABLE_GSS
+	memcpy(st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+#endif
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/*
+ * Finalizes the startup pgstat entry by filling in the user/database IDs,
+ * clearing STATE_STARTING, and reporting the application_name.
+ *
+ * We must be inside a transaction if this isn't an aux process, as we may need
+ * to do encoding conversion.
+ */
+void
+pgstat_bestart_final(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	Oid			userid;
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+
+	/* We have userid for client-backends, wal-sender and bgworker processes */
+	if (MyBackendType == B_BACKEND
+		|| MyBackendType == B_WAL_SENDER
+		|| MyBackendType == B_BG_WORKER)
+		userid = GetSessionUserId();
+	else
+		userid = InvalidOid;
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_databaseid = MyDatabaseId;
+	beentry->st_userid = userid;
+	beentry->st_state = STATE_UNDEFINED;
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
 
 	/* Update app name to current GUC setting */
+	/* TODO: ask the list: maybe do this before setting STATE_UNDEFINED? */
 	if (application_name)
 		pgstat_report_appname(application_name);
 }
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index f7b50e0b5a..c461bbd400 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -366,6 +366,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index a024b1151d..c301d0b2b7 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -714,6 +715,23 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That
+	 * way, if LWLocks or third-party authentication should happen to hang,
+	 * the DBA will still be able to see what's going on.
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_initial();
+		if (MyProcPort)
+			pgstat_bestart_security();	/* fill in any SSL/GSS info too */
+
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -782,9 +800,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -805,8 +820,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* The autovacuum launcher is done here */
 	if (AmAutoVacuumLauncherProcess())
 	{
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		return;
 	}
@@ -882,6 +897,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
@@ -889,6 +905,12 @@ InitPostgres(const char *in_dbname, Oid dboid,
 			InitializeSystemUser(MyClientConnectionInfo.authn_id,
 								 hba_authname(MyClientConnectionInfo.auth_method));
 		am_superuser = superuser();
+
+		/*
+		 * Authentication may have changed SSL/GSS details for the session, so
+		 * report it again.
+		 */
+		pgstat_bestart_security();
 	}
 
 	/*
@@ -961,8 +983,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		/* initialize client encoding */
 		InitializeClientEncoding();
 
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		/* close the transaction we started above */
 		CommitTransactionCommand();
@@ -1005,7 +1027,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		 */
 		if (!bootstrap)
 		{
-			pgstat_bestart();
+			pgstat_bestart_final();
 			CommitTransactionCommand();
 		}
 		return;
@@ -1205,9 +1227,9 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	if ((flags & INIT_PG_LOAD_SESSION_LIBS) != 0)
 		process_session_preload_libraries();
 
-	/* report this backend in the PgBackendStatus array */
+	/* fill in the remainder of the PgBackendStatus array */
 	if (!bootstrap)
-		pgstat_bestart();
+		pgstat_bestart_final();
 
 	/* close the transaction we started above */
 	if (!bootstrap)
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index 97874300c3..ac693b7d8b 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,7 +310,9 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
-extern void pgstat_bestart(void);
+extern void pgstat_bestart_initial(void);
+extern void pgstat_bestart_security(void);
+extern void pgstat_bestart_final(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
 
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index da0b71873a..d05b15de7c 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,10 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+EXTRA_INSTALL = src/test/modules/injection_points
+
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index 8f5688dcc1..fe4ca51873 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_pre_auth.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
new file mode 100644
index 0000000000..bb7f61a7ca
--- /dev/null
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -0,0 +1,81 @@
+
+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
+
+# Tests for connection behavior prior to authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(100_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(100_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
diff --git a/src/test/ssl/Makefile b/src/test/ssl/Makefile
index 6441a8047b..880fe656c7 100644
--- a/src/test/ssl/Makefile
+++ b/src/test/ssl/Makefile
@@ -10,12 +10,13 @@
 #-------------------------------------------------------------------------
 
 EXTRA_INSTALL = contrib/sslinfo
+EXTRA_INSTALL += src/test/modules/injection_points
 
 subdir = src/test/ssl
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-export OPENSSL with_ssl
+export OPENSSL enable_injection_points with_ssl
 
 # The sslfiles targets are separated into their own file due to interactions
 # with settings in Makefile.global.
diff --git a/src/test/ssl/meson.build b/src/test/ssl/meson.build
index b3c5503f79..b7827c72fa 100644
--- a/src/test/ssl/meson.build
+++ b/src/test/ssl/meson.build
@@ -6,6 +6,7 @@ tests += {
   'bd': meson.current_build_dir(),
   'tap': {
     'env': {
+      'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
       'with_ssl': ssl_library,
       'OPENSSL': openssl.found() ? openssl.path() : '',
     },
diff --git a/src/test/ssl/t/001_ssltests.pl b/src/test/ssl/t/001_ssltests.pl
index 131460a1fe..7d7d071614 100644
--- a/src/test/ssl/t/001_ssltests.pl
+++ b/src/test/ssl/t/001_ssltests.pl
@@ -7,6 +7,7 @@ use Config qw ( %Config );
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
+use Time::HiRes qw(usleep);
 
 use FindBin;
 use lib $FindBin::RealBin;
@@ -71,6 +72,23 @@ $node->start;
 my $result = $node->safe_psql('postgres', "SHOW ssl_library");
 is($result, $ssl_server->ssl_library(), 'ssl_library parameter');
 
+my $injection_points_unavailable = '';
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	$injection_points_unavailable =
+	  'Injection points not supported by this build';
+}
+elsif (!$node->check_extension('injection_points'))
+{
+	$injection_points_unavailable =
+	  'Extension injection_points not installed';
+}
+else
+{
+	# For ease of setup, make injection_points available for all new databases.
+	$node->safe_psql('template1', 'CREATE EXTENSION injection_points');
+}
+
 $ssl_server->configure_test_server_for_ssl($node, $SERVERHOSTADDR,
 	$SERVERHOSTCIDR, 'trust');
 
@@ -554,6 +572,53 @@ command_like(
 				^\d+,t,TLSv[\d.]+,[\w-]+,\d+,_null_,_null_,_null_\r?$}mx,
 	'pg_stat_ssl view without client certificate');
 
+# Test that pg_stat_ssl gets filled in early, prior to authentication. Requires
+# injection point support.
+SKIP:
+{
+	skip $injection_points_unavailable, 1 if $injection_points_unavailable;
+
+	# Connect to the server and inject a waitpoint.
+	my $psql =
+	  $node->background_psql('trustdb', connstr => "$common_connstr user=");
+	$psql->query_safe(
+		"SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+	# From this point on, all new connections will hang during startup, just
+	# before authentication. Use the $psql connection handle for server
+	# interaction.
+	my $conn = $node->background_psql(
+		'trustdb',
+		connstr => $common_connstr,
+		wait => 0);
+
+	# Wait for the connection to show up.
+	my $pid;
+	while (1)
+	{
+		$pid = $psql->query(
+			"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+		last if $pid ne "";
+
+		usleep(100_000);
+	}
+
+	like(
+		$psql->query(
+			"SELECT ssl, version, cipher, bits FROM pg_stat_ssl WHERE pid = $pid"
+		),
+		qr/^t|TLSv[\d.]+|[\w-]+|\d+$/,
+		'pg_stat_ssl view is updated prior to authentication');
+
+	# Detach the waitpoint and wait for the connection to complete.
+	$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+	$conn->wait_connect();
+
+	$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+	$psql->quit();
+	$conn->quit();
+}
+
 # Test min/max SSL protocol versions.
 $node->connect_ok(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require ssl_min_protocol_version=TLSv1.2 ssl_max_protocol_version=TLSv1.2",
-- 
2.34.1

From 0cb5ab8b89d3595e32c8cd86fa49376e1a482493 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v7 2/3] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 56 +++++++++++++++++--
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 27 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 81a4a95152..a148e63711 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 47e8c91606..fe4cbeb8a0 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -1000,6 +1001,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1011,6 +1013,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1222,6 +1225,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1231,6 +1235,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1296,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1305,6 +1312,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1402,11 +1410,13 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
@@ -1503,8 +1513,11 @@ pg_SSPI_make_upn(char *accountname,
 	 */
 
 	samname = psprintf("%s\\%s", domainname, accountname);
+
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						NULL, &upnamesize);
+	pgstat_report_wait_end();
 
 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
 		|| upnamesize == 0)
@@ -1519,8 +1532,10 @@ pg_SSPI_make_upn(char *accountname,
 	/* upnamesize includes the terminating NUL. */
 	upname = palloc(upnamesize);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						upname, &upnamesize);
+	pgstat_report_wait_end();
 
 	pfree(samname);
 	if (res)
@@ -2119,7 +2134,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2132,7 +2149,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2272,7 +2291,11 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 			}
 
 			/* Look up a list of LDAP server hosts and port numbers */
-			if (ldap_domain2hostlist(domain, &hostlist))
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
+			r = ldap_domain2hostlist(domain, &hostlist);
+			pgstat_report_wait_end();
+
+			if (r)
 			{
 				ereport(LOG,
 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
@@ -2366,11 +2389,15 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 
 	if (port->hba->ldaptls)
 	{
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
 #ifndef WIN32
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL);
 #else
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
 #endif
+		pgstat_report_wait_end();
+
+		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
 					(errmsg("could not start LDAP TLS session: %s",
@@ -2530,9 +2557,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2555,6 +2585,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2562,6 +2594,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2630,7 +2663,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -3079,8 +3114,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		return STATUS_ERROR;
 	}
 
-	if (sendto(sock, radius_buffer, packetlength, 0,
-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
+	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
+	r = sendto(sock, radius_buffer, packetlength, 0,
+			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
+	pgstat_report_wait_end();
+
+	if (r < 0)
 	{
 		ereport(LOG,
 				(errmsg("could not send RADIUS packet: %m")));
@@ -3128,7 +3167,10 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		FD_ZERO(&fdset);
 		FD_SET(sock, &fdset);
 
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
+		pgstat_report_wait_end();
+
 		if (r < 0)
 		{
 			if (errno == EINTR)
@@ -3161,8 +3203,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 
 		addrsize = sizeof(remoteaddr);
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
 								(struct sockaddr *) &remoteaddr, &addrsize);
+		pgstat_report_wait_end();
+
 		if (packetlength < 0)
 		{
 			ereport(LOG,
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d930277140..a388999b1a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index 16144c2b72..db8474bc85 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -161,6 +161,33 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
+RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
+RADIUS_WAIT	"Waiting for a RADIUS server to respond."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 9f18a753d4..014d536441 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index fad7fc3a7e..030d1a8321 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -179,6 +179,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -187,7 +188,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

From a0308928dd3c6dbb83c43f74bb3913f9bccf8596 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 8 Nov 2024 14:19:26 -0800
Subject: [PATCH v7 3/3] squash! Report external auth calls as wait events

Add a wait event around all calls to ldap_unbind().
---
 src/backend/libpq/auth.c                      | 33 ++++++++++++++-----
 .../utils/activity/wait_event_names.txt       |  1 +
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index fe4cbeb8a0..df38d66f48 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -2226,6 +2226,7 @@ CheckBSDAuth(Port *port, char *user)
 #ifdef USE_LDAP
 
 static int	errdetail_for_ldap(LDAP *ldap);
+static void unbind(LDAP *ldap);
 
 /*
  * Initialize a connection to the LDAP server, including setting up
@@ -2383,7 +2384,7 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 				(errmsg("could not set LDAP protocol version: %s",
 						ldap_err2string(r)),
 				 errdetail_for_ldap(*ldap)));
-		ldap_unbind(*ldap);
+		unbind(*ldap);
 		return STATUS_ERROR;
 	}
 
@@ -2403,7 +2404,7 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 					(errmsg("could not start LDAP TLS session: %s",
 							ldap_err2string(r)),
 					 errdetail_for_ldap(*ldap)));
-			ldap_unbind(*ldap);
+			unbind(*ldap);
 			return STATUS_ERROR;
 		}
 	}
@@ -2547,7 +2548,7 @@ CheckLDAPAuth(Port *port)
 			{
 				ereport(LOG,
 						(errmsg("invalid character in user name for LDAP authentication")));
-				ldap_unbind(ldap);
+				unbind(ldap);
 				pfree(passwd);
 				return STATUS_ERROR;
 			}
@@ -2571,7 +2572,7 @@ CheckLDAPAuth(Port *port)
 							server_name,
 							ldap_err2string(r)),
 					 errdetail_for_ldap(ldap)));
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			return STATUS_ERROR;
 		}
@@ -2604,7 +2605,7 @@ CheckLDAPAuth(Port *port)
 					 errdetail_for_ldap(ldap)));
 			if (search_message != NULL)
 				ldap_msgfree(search_message);
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			pfree(filter);
 			return STATUS_ERROR;
@@ -2626,7 +2627,7 @@ CheckLDAPAuth(Port *port)
 										  count,
 										  filter, server_name, count)));
 
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2645,7 +2646,7 @@ CheckLDAPAuth(Port *port)
 							filter, server_name,
 							ldap_err2string(error)),
 					 errdetail_for_ldap(ldap)));
-			ldap_unbind(ldap);
+			unbind(ldap);
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2673,7 +2674,7 @@ CheckLDAPAuth(Port *port)
 				(errmsg("LDAP login failed for user \"%s\" on server \"%s\": %s",
 						fulluser, server_name, ldap_err2string(r)),
 				 errdetail_for_ldap(ldap)));
-		ldap_unbind(ldap);
+		unbind(ldap);
 		pfree(passwd);
 		pfree(fulluser);
 		return STATUS_ERROR;
@@ -2682,7 +2683,7 @@ CheckLDAPAuth(Port *port)
 	/* Save the original bind DN as the authenticated identity. */
 	set_authn_id(port, fulluser);
 
-	ldap_unbind(ldap);
+	unbind(ldap);
 	pfree(passwd);
 	pfree(fulluser);
 
@@ -2709,6 +2710,20 @@ errdetail_for_ldap(LDAP *ldap)
 	return 0;
 }
 
+/*
+ * ldap_unbind() is not a lightweight operation; it writes data to the socket
+ * and may need to tear down (LDAP) SASL mechanisms, depending on the LDAP
+ * implementation in use. This function just wraps the unbind with pgstat
+ * reporting in case something takes longer than expected.
+ */
+static void
+unbind(LDAP *ldap)
+{
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND);
+	ldap_unbind(ldap);
+	pgstat_report_wait_end();
+}
+
 #endif							/* USE_LDAP */
 
 
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index db8474bc85..00a9459ad4 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -176,6 +176,7 @@ LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory
 LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
 LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
 LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+LDAP_UNBIND	"Waiting for an LDAP connection to be unbound and closed."
 PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
 PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
 RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
-- 
2.34.1

1:  b91a602cab8 ! 1:  81a61854bdf pgstat: report in earlier with STATE_STARTING
    @@ src/backend/utils/activity/backend_status.c: pgstat_bestart(void)
     +
     +	PGSTAT_END_WRITE_ACTIVITY(beentry);
      
    + 	/* Create the backend statistics entry */
    + 	if (pgstat_tracks_backend_bktype(MyBackendType))
    + 		pgstat_create_backend(MyProcNumber);
    + 
      	/* Update app name to current GUC setting */
     +	/* TODO: ask the list: maybe do this before setting STATE_UNDEFINED? */
      	if (application_name)
    @@ src/test/authentication/meson.build: tests += {
      ## src/test/authentication/t/007_pre_auth.pl (new) ##
     @@
     +
    -+# Copyright (c) 2021-2024, PostgreSQL Global Development Group
    ++# Copyright (c) 2021-2025, PostgreSQL Global Development Group
     +
     +# Tests for connection behavior prior to authentication.
     +
    @@ src/test/ssl/t/001_ssltests.pl: command_like(
     +	while (1)
     +	{
     +		$pid = $psql->query(
    -+			"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
    ++			"SELECT pid FROM pg_stat_activity WHERE state = 'starting' AND client_addr IS NOT NULL;");
     +		last if $pid ne "";
     +
     +		usleep(100_000);
    @@ src/test/ssl/t/001_ssltests.pl: command_like(
     +		$psql->query(
     +			"SELECT ssl, version, cipher, bits FROM pg_stat_ssl WHERE pid = $pid"
     +		),
    -+		qr/^t|TLSv[\d.]+|[\w-]+|\d+$/,
    ++		qr/^t\|TLSv[\d.]+\|[\w-]+\|\d+$/,
     +		'pg_stat_ssl view is updated prior to authentication');
     +
     +	# Detach the waitpoint and wait for the connection to complete.
2:  0cb5ab8b89d = 2:  e734e46009f Report external auth calls as wait events
3:  a0308928dd3 < -:  ----------- squash! Report external auth calls as wait events
-:  ----------- > 3:  39c7d9ce42b squash! Report external auth calls as wait events

From 81a61854bdf419d045c4b82896b8889938e41184 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v8 1/3] pgstat: report in earlier with STATE_STARTING

Split pgstat_bestart() into three phases for better observability:

1) pgstat_bestart_initial() reports a 'starting' state while waiting for
   backend initialization and client authentication to complete.  Since
   we hold a transaction open for a good amount of that, and some
   authentication methods call out to external systems, having an early
   pg_stat_activity entry helps DBAs debug when things go badly wrong.

2) pgstat_bestart_security() reports the SSL/GSS status of the
   connection.  Some backends don't call this at all; others call it
   twice, once after transport establishment and once after client
   authentication.

3) pgstat_bestart_final() reports the user and database IDs, takes the
   entry out of STATE_STARTING, and reports the application_name.
   TODO: should the order of those last two be swapped?
---
 doc/src/sgml/monitoring.sgml                |   6 +
 src/backend/postmaster/auxprocess.c         |   3 +-
 src/backend/utils/activity/backend_status.c | 207 +++++++++++++-------
 src/backend/utils/adt/pgstatfuncs.c         |   3 +
 src/backend/utils/init/postinit.c           |  42 +++-
 src/include/utils/backend_status.h          |   5 +-
 src/test/authentication/Makefile            |   4 +
 src/test/authentication/meson.build         |   4 +
 src/test/authentication/t/007_pre_auth.pl   |  81 ++++++++
 src/test/ssl/Makefile                       |   3 +-
 src/test/ssl/meson.build                    |   1 +
 src/test/ssl/t/001_ssltests.pl              |  65 ++++++
 12 files changed, 335 insertions(+), 89 deletions(-)
 create mode 100644 src/test/authentication/t/007_pre_auth.pl

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index edc2470bcf9..dcb2d77adec 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
diff --git a/src/backend/postmaster/auxprocess.c b/src/backend/postmaster/auxprocess.c
index ff366ceb0fc..4f6795f7265 100644
--- a/src/backend/postmaster/auxprocess.c
+++ b/src/backend/postmaster/auxprocess.c
@@ -78,7 +78,8 @@ AuxiliaryProcessMainCommon(void)
 
 	/* Initialize backend status information */
 	pgstat_beinit();
-	pgstat_bestart();
+	pgstat_bestart_initial();
+	pgstat_bestart_final();
 
 	/* register a before-shutdown callback for LWLock cleanup */
 	before_shmem_exit(ShutdownAuxiliaryProcess, 0);
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 731342799a6..f08b41054bb 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -253,31 +253,18 @@ pgstat_beinit(void)
 	on_shmem_exit(pgstat_beshutdown_hook, 0);
 }
 
-
-/* ----------
- * pgstat_bestart() -
- *
- *	Initialize this backend's entry in the PgBackendStatus array.
- *	Called from InitPostgres.
- *
- *	Apart from auxiliary processes, MyDatabaseId, session userid, and
- *	application_name must already be set (hence, this cannot be combined
- *	with pgstat_beinit).  Note also that we must be inside a transaction
- *	if this isn't an aux process, as we may need to do encoding conversion
- *	on some strings.
- *----------
+/*
+ * Clears out a new pgstat entry, initializing it to suitable defaults and
+ * reporting STATE_STARTING. Backends should continue filling in any transport
+ * security details as needed with pgstat_bestart_security(), and must finally
+ * exit STATE_STARTING by calling pgstat_bestart_final(), once user and database
+ * IDs have been determined.
  */
 void
-pgstat_bestart(void)
+pgstat_bestart_initial(void)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
-#ifdef USE_SSL
-	PgBackendSSLStatus lsslstatus;
-#endif
-#ifdef ENABLE_GSS
-	PgBackendGSSStatus lgssstatus;
-#endif
 
 	/* pgstats state must be initialized from pgstat_beinit() */
 	Assert(vbeentry != NULL);
@@ -297,14 +284,6 @@ pgstat_bestart(void)
 		   unvolatize(PgBackendStatus *, vbeentry),
 		   sizeof(PgBackendStatus));
 
-	/* These structs can just start from zeroes each time, though */
-#ifdef USE_SSL
-	memset(&lsslstatus, 0, sizeof(lsslstatus));
-#endif
-#ifdef ENABLE_GSS
-	memset(&lgssstatus, 0, sizeof(lgssstatus));
-#endif
-
 	/*
 	 * Now fill in all the fields of lbeentry, except for strings that are
 	 * out-of-line data.  Those have to be handled separately, below.
@@ -315,15 +294,8 @@ pgstat_bestart(void)
 	lbeentry.st_activity_start_timestamp = 0;
 	lbeentry.st_state_start_timestamp = 0;
 	lbeentry.st_xact_start_timestamp = 0;
-	lbeentry.st_databaseid = MyDatabaseId;
-
-	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
-		lbeentry.st_userid = GetSessionUserId();
-	else
-		lbeentry.st_userid = InvalidOid;
+	lbeentry.st_databaseid = InvalidOid;
+	lbeentry.st_userid = InvalidOid;
 
 	/*
 	 * We may not have a MyProcPort (eg, if this is the autovacuum process).
@@ -336,46 +308,10 @@ pgstat_bestart(void)
 	else
 		MemSet(&lbeentry.st_clientaddr, 0, sizeof(lbeentry.st_clientaddr));
 
-#ifdef USE_SSL
-	if (MyProcPort && MyProcPort->ssl_in_use)
-	{
-		lbeentry.st_ssl = true;
-		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
-		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
-		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
-		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
-		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
-		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_ssl = false;
-	}
-#else
 	lbeentry.st_ssl = false;
-#endif
-
-#ifdef ENABLE_GSS
-	if (MyProcPort && MyProcPort->gss != NULL)
-	{
-		const char *princ = be_gssapi_get_princ(MyProcPort);
-
-		lbeentry.st_gss = true;
-		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
-		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
-		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
-		if (princ)
-			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_gss = false;
-	}
-#else
 	lbeentry.st_gss = false;
-#endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = STATE_STARTING;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
@@ -417,20 +353,139 @@ pgstat_bestart(void)
 	lbeentry.st_clienthostname[NAMEDATALEN - 1] = '\0';
 	lbeentry.st_activity_raw[pgstat_track_activity_query_size - 1] = '\0';
 
+	/* These structs can just start from zeroes each time */
 #ifdef USE_SSL
-	memcpy(lbeentry.st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+	memset(lbeentry.st_sslstatus, 0, sizeof(PgBackendSSLStatus));
 #endif
 #ifdef ENABLE_GSS
-	memcpy(lbeentry.st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+	memset(lbeentry.st_gssstatus, 0, sizeof(PgBackendGSSStatus));
 #endif
 
 	PGSTAT_END_WRITE_ACTIVITY(vbeentry);
+}
+
+/*
+ * Fill in SSL and GSS information for the pgstat entry. This is separate from
+ * pgstat_bestart_initial() so that backends may call it multiple times as
+ * security details are filled in.
+ *
+ * This should only be called from backends with a MyProcPort.
+ */
+void
+pgstat_bestart_security(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	bool		ssl = false;
+	bool		gss = false;
+#ifdef USE_SSL
+	PgBackendSSLStatus lsslstatus;
+	PgBackendSSLStatus *st_sslstatus;
+#endif
+#ifdef ENABLE_GSS
+	PgBackendGSSStatus lgssstatus;
+	PgBackendGSSStatus *st_gssstatus;
+#endif
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+	Assert(MyProcPort);			/* otherwise there's no point */
+
+#ifdef USE_SSL
+	st_sslstatus = beentry->st_sslstatus;
+	memset(&lsslstatus, 0, sizeof(lsslstatus));
+
+	if (MyProcPort->ssl_in_use)
+	{
+		ssl = true;
+		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
+		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
+		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
+		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
+		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
+		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
+	}
+#endif
+
+#ifdef ENABLE_GSS
+	st_gssstatus = beentry->st_gssstatus;
+	memset(&lgssstatus, 0, sizeof(lgssstatus));
+
+	if (MyProcPort->gss != NULL)
+	{
+		const char *princ = be_gssapi_get_princ(MyProcPort);
+
+		gss = true;
+		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
+		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
+		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
+		if (princ)
+			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
+	}
+#endif
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_ssl = ssl;
+	beentry->st_gss = gss;
+
+#ifdef USE_SSL
+	memcpy(st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+#endif
+#ifdef ENABLE_GSS
+	memcpy(st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+#endif
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/*
+ * Finalizes the startup pgstat entry by filling in the user/database IDs,
+ * clearing STATE_STARTING, and reporting the application_name.
+ *
+ * We must be inside a transaction if this isn't an aux process, as we may need
+ * to do encoding conversion.
+ */
+void
+pgstat_bestart_final(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	Oid			userid;
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+
+	/* We have userid for client-backends, wal-sender and bgworker processes */
+	if (MyBackendType == B_BACKEND
+		|| MyBackendType == B_WAL_SENDER
+		|| MyBackendType == B_BG_WORKER)
+		userid = GetSessionUserId();
+	else
+		userid = InvalidOid;
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_databaseid = MyDatabaseId;
+	beentry->st_userid = userid;
+	beentry->st_state = STATE_UNDEFINED;
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
 
 	/* Create the backend statistics entry */
 	if (pgstat_tracks_backend_bktype(MyBackendType))
 		pgstat_create_backend(MyProcNumber);
 
 	/* Update app name to current GUC setting */
+	/* TODO: ask the list: maybe do this before setting STATE_UNDEFINED? */
 	if (application_name)
 		pgstat_report_appname(application_name);
 }
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index e9096a88492..c754a31253d 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -393,6 +393,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index 01bb6a410cb..62364f6b767 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -58,6 +58,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -717,6 +718,23 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That
+	 * way, if LWLocks or third-party authentication should happen to hang,
+	 * the DBA will still be able to see what's going on.
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_initial();
+		if (MyProcPort)
+			pgstat_bestart_security();	/* fill in any SSL/GSS info too */
+
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -785,9 +803,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -808,8 +823,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* The autovacuum launcher is done here */
 	if (AmAutoVacuumLauncherProcess())
 	{
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		return;
 	}
@@ -874,6 +889,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
@@ -881,6 +897,12 @@ InitPostgres(const char *in_dbname, Oid dboid,
 			InitializeSystemUser(MyClientConnectionInfo.authn_id,
 								 hba_authname(MyClientConnectionInfo.auth_method));
 		am_superuser = superuser();
+
+		/*
+		 * Authentication may have changed SSL/GSS details for the session, so
+		 * report it again.
+		 */
+		pgstat_bestart_security();
 	}
 
 	/*
@@ -952,8 +974,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		/* initialize client encoding */
 		InitializeClientEncoding();
 
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		/* close the transaction we started above */
 		CommitTransactionCommand();
@@ -996,7 +1018,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		 */
 		if (!bootstrap)
 		{
-			pgstat_bestart();
+			pgstat_bestart_final();
 			CommitTransactionCommand();
 		}
 		return;
@@ -1196,9 +1218,9 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	if ((flags & INIT_PG_LOAD_SESSION_LIBS) != 0)
 		process_session_preload_libraries();
 
-	/* report this backend in the PgBackendStatus array */
+	/* fill in the remainder of the PgBackendStatus array */
 	if (!bootstrap)
-		pgstat_bestart();
+		pgstat_bestart_final();
 
 	/* close the transaction we started above */
 	if (!bootstrap)
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index d3d4ff6c5c9..1c9b4fe14d0 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,7 +310,9 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
-extern void pgstat_bestart(void);
+extern void pgstat_bestart_initial(void);
+extern void pgstat_bestart_security(void);
+extern void pgstat_bestart_final(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
 
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index c4022dc829b..8b5beced080 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,10 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+EXTRA_INSTALL = src/test/modules/injection_points
+
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index f6e48411c11..800b3a5ff40 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_pre_auth.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
new file mode 100644
index 00000000000..a638226dbaf
--- /dev/null
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -0,0 +1,81 @@
+
+# Copyright (c) 2021-2025, PostgreSQL Global Development Group
+
+# Tests for connection behavior prior to authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(100_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(100_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
diff --git a/src/test/ssl/Makefile b/src/test/ssl/Makefile
index e8a1639db2d..2d800c4a254 100644
--- a/src/test/ssl/Makefile
+++ b/src/test/ssl/Makefile
@@ -10,12 +10,13 @@
 #-------------------------------------------------------------------------
 
 EXTRA_INSTALL = contrib/sslinfo
+EXTRA_INSTALL += src/test/modules/injection_points
 
 subdir = src/test/ssl
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-export OPENSSL with_ssl
+export OPENSSL enable_injection_points with_ssl
 
 # The sslfiles targets are separated into their own file due to interactions
 # with settings in Makefile.global.
diff --git a/src/test/ssl/meson.build b/src/test/ssl/meson.build
index cf8b2b9303a..5ba8bdb373d 100644
--- a/src/test/ssl/meson.build
+++ b/src/test/ssl/meson.build
@@ -6,6 +6,7 @@ tests += {
   'bd': meson.current_build_dir(),
   'tap': {
     'env': {
+      'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
       'with_ssl': ssl_library,
       'OPENSSL': openssl.found() ? openssl.path() : '',
     },
diff --git a/src/test/ssl/t/001_ssltests.pl b/src/test/ssl/t/001_ssltests.pl
index 5422511d4ab..4d1ab3493db 100644
--- a/src/test/ssl/t/001_ssltests.pl
+++ b/src/test/ssl/t/001_ssltests.pl
@@ -7,6 +7,7 @@ use Config qw ( %Config );
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
+use Time::HiRes qw(usleep);
 
 use FindBin;
 use lib $FindBin::RealBin;
@@ -71,6 +72,23 @@ $node->start;
 my $result = $node->safe_psql('postgres', "SHOW ssl_library");
 is($result, $ssl_server->ssl_library(), 'ssl_library parameter');
 
+my $injection_points_unavailable = '';
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	$injection_points_unavailable =
+	  'Injection points not supported by this build';
+}
+elsif (!$node->check_extension('injection_points'))
+{
+	$injection_points_unavailable =
+	  'Extension injection_points not installed';
+}
+else
+{
+	# For ease of setup, make injection_points available for all new databases.
+	$node->safe_psql('template1', 'CREATE EXTENSION injection_points');
+}
+
 $ssl_server->configure_test_server_for_ssl($node, $SERVERHOSTADDR,
 	$SERVERHOSTCIDR, 'trust');
 
@@ -556,6 +574,53 @@ command_like(
 				^\d+,t,TLSv[\d.]+,[\w-]+,\d+,_null_,_null_,_null_\r?$}mx,
 	'pg_stat_ssl view without client certificate');
 
+# Test that pg_stat_ssl gets filled in early, prior to authentication. Requires
+# injection point support.
+SKIP:
+{
+	skip $injection_points_unavailable, 1 if $injection_points_unavailable;
+
+	# Connect to the server and inject a waitpoint.
+	my $psql =
+	  $node->background_psql('trustdb', connstr => "$common_connstr user=");
+	$psql->query_safe(
+		"SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+	# From this point on, all new connections will hang during startup, just
+	# before authentication. Use the $psql connection handle for server
+	# interaction.
+	my $conn = $node->background_psql(
+		'trustdb',
+		connstr => $common_connstr,
+		wait => 0);
+
+	# Wait for the connection to show up.
+	my $pid;
+	while (1)
+	{
+		$pid = $psql->query(
+			"SELECT pid FROM pg_stat_activity WHERE state = 'starting' AND client_addr IS NOT NULL;");
+		last if $pid ne "";
+
+		usleep(100_000);
+	}
+
+	like(
+		$psql->query(
+			"SELECT ssl, version, cipher, bits FROM pg_stat_ssl WHERE pid = $pid"
+		),
+		qr/^t\|TLSv[\d.]+\|[\w-]+\|\d+$/,
+		'pg_stat_ssl view is updated prior to authentication');
+
+	# Detach the waitpoint and wait for the connection to complete.
+	$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+	$conn->wait_connect();
+
+	$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+	$psql->quit();
+	$conn->quit();
+}
+
 # Test min/max SSL protocol versions.
 $node->connect_ok(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require ssl_min_protocol_version=TLSv1.2 ssl_max_protocol_version=TLSv1.2",
-- 
2.34.1

From e734e46009f35e27bcfc05d8f75546eb7489131d Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v8 2/3] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 56 +++++++++++++++++--
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 27 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index dcb2d77adec..33f67c00100 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index d6ef32cc823..f79626f019f 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -38,6 +38,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -984,6 +985,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -995,6 +997,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1206,6 +1209,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1215,6 +1219,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1280,6 +1286,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1289,6 +1296,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1386,11 +1394,13 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
@@ -1487,8 +1497,11 @@ pg_SSPI_make_upn(char *accountname,
 	 */
 
 	samname = psprintf("%s\\%s", domainname, accountname);
+
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						NULL, &upnamesize);
+	pgstat_report_wait_end();
 
 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
 		|| upnamesize == 0)
@@ -1503,8 +1516,10 @@ pg_SSPI_make_upn(char *accountname,
 	/* upnamesize includes the terminating NUL. */
 	upname = palloc(upnamesize);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						upname, &upnamesize);
+	pgstat_report_wait_end();
 
 	pfree(samname);
 	if (res)
@@ -2103,7 +2118,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2116,7 +2133,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2256,7 +2275,11 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 			}
 
 			/* Look up a list of LDAP server hosts and port numbers */
-			if (ldap_domain2hostlist(domain, &hostlist))
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
+			r = ldap_domain2hostlist(domain, &hostlist);
+			pgstat_report_wait_end();
+
+			if (r)
 			{
 				ereport(LOG,
 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
@@ -2350,11 +2373,15 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 
 	if (port->hba->ldaptls)
 	{
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
 #ifndef WIN32
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL);
 #else
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
 #endif
+		pgstat_report_wait_end();
+
+		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
 					(errmsg("could not start LDAP TLS session: %s",
@@ -2514,9 +2541,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2539,6 +2569,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2546,6 +2578,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2614,7 +2647,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -3063,8 +3098,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		return STATUS_ERROR;
 	}
 
-	if (sendto(sock, radius_buffer, packetlength, 0,
-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
+	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
+	r = sendto(sock, radius_buffer, packetlength, 0,
+			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
+	pgstat_report_wait_end();
+
+	if (r < 0)
 	{
 		ereport(LOG,
 				(errmsg("could not send RADIUS packet: %m")));
@@ -3112,7 +3151,10 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		FD_ZERO(&fdset);
 		FD_SET(sock, &fdset);
 
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
+		pgstat_report_wait_end();
+
 		if (r < 0)
 		{
 			if (errno == EINTR)
@@ -3145,8 +3187,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 
 		addrsize = sizeof(remoteaddr);
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
 								(struct sockaddr *) &remoteaddr, &addrsize);
+		pgstat_report_wait_end();
+
 		if (packetlength < 0)
 		{
 			ereport(LOG,
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d9b8f34a355..6cc3e1e7c7a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index e199f071628..a2852225614 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -162,6 +162,33 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
+RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
+RADIUS_WAIT	"Waiting for a RADIUS server to respond."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b8cb3e5a430..3d995a9e5be 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index 352abc0bd42..7e671563d68 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -180,6 +180,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -188,7 +189,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

From 39c7d9ce42be3f0878ee3c70b9b17c3ca96f9321 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 8 Nov 2024 14:19:26 -0800
Subject: [PATCH v8 3/3] squash! Report external auth calls as wait events

Add a wait event around all calls to ldap_unbind(). (For the record, I
do not want to implement this in this way.)
---
 src/backend/libpq/auth.c                       | 18 ++++++++++++++++++
 .../utils/activity/wait_event_names.txt        |  9 +++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index f79626f019f..e42e7daa29c 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -2367,7 +2367,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 				(errmsg("could not set LDAP protocol version: %s",
 						ldap_err2string(r)),
 				 errdetail_for_ldap(*ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SET_OPTION);
 		ldap_unbind(*ldap);
+		pgstat_report_wait_end();
 		return STATUS_ERROR;
 	}
 
@@ -2387,7 +2389,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 					(errmsg("could not start LDAP TLS session: %s",
 							ldap_err2string(r)),
 					 errdetail_for_ldap(*ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_START_TLS);
 			ldap_unbind(*ldap);
+			pgstat_report_wait_end();
 			return STATUS_ERROR;
 		}
 	}
@@ -2531,7 +2535,9 @@ CheckLDAPAuth(Port *port)
 			{
 				ereport(LOG,
 						(errmsg("invalid character in user name for LDAP authentication")));
+				pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_NAME_CHECK);
 				ldap_unbind(ldap);
+				pgstat_report_wait_end();
 				pfree(passwd);
 				return STATUS_ERROR;
 			}
@@ -2555,7 +2561,9 @@ CheckLDAPAuth(Port *port)
 							server_name,
 							ldap_err2string(r)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND_FOR_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			return STATUS_ERROR;
 		}
@@ -2588,7 +2596,9 @@ CheckLDAPAuth(Port *port)
 					 errdetail_for_ldap(ldap)));
 			if (search_message != NULL)
 				ldap_msgfree(search_message);
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			return STATUS_ERROR;
@@ -2610,7 +2620,9 @@ CheckLDAPAuth(Port *port)
 										  count,
 										  filter, server_name, count)));
 
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_COUNT_ENTRIES);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2629,7 +2641,9 @@ CheckLDAPAuth(Port *port)
 							filter, server_name,
 							ldap_err2string(error)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_GET_DN);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2657,7 +2671,9 @@ CheckLDAPAuth(Port *port)
 				(errmsg("LDAP login failed for user \"%s\" on server \"%s\": %s",
 						fulluser, server_name, ldap_err2string(r)),
 				 errdetail_for_ldap(ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND);
 		ldap_unbind(ldap);
+		pgstat_report_wait_end();
 		pfree(passwd);
 		pfree(fulluser);
 		return STATUS_ERROR;
@@ -2666,7 +2682,9 @@ CheckLDAPAuth(Port *port)
 	/* Save the original bind DN as the authenticated identity. */
 	set_authn_id(port, fulluser);
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_SUCCESS);
 	ldap_unbind(ldap);
+	pgstat_report_wait_end();
 	pfree(passwd);
 	pfree(fulluser);
 
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index a2852225614..f082756c294 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -177,6 +177,15 @@ LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory
 LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
 LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
 LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+LDAP_UNBIND_AFTER_BIND	"Waiting for an LDAP connection to be unbound after a simple bind failed."
+LDAP_UNBIND_AFTER_BIND_FOR_SEARCH	"Waiting for an LDAP connection to be unbound after a bind for search failed."
+LDAP_UNBIND_AFTER_COUNT_ENTRIES	"Waiting for an LDAP connection to be unbound after an entry count failed."
+LDAP_UNBIND_AFTER_GET_DN	"Waiting for an LDAP connection to be unbound after ldap_get_dn failed."
+LDAP_UNBIND_AFTER_NAME_CHECK	"Waiting for an LDAP connection to be unbound after a name check failed."
+LDAP_UNBIND_AFTER_SEARCH	"Waiting for an LDAP connection to be unbound after a bind+search failed."
+LDAP_UNBIND_AFTER_SET_OPTION	"Waiting for an LDAP connection to be unbound after ldap_set_option failed."
+LDAP_UNBIND_AFTER_START_TLS	"Waiting for an LDAP connection to be unbound after ldap_start_tls_s failed."
+LDAP_UNBIND_SUCCESS	"Waiting for a successful LDAP connection to be unbound."
 PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
 PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
 RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
-- 
2.34.1

1:  81a61854bdf ! 1:  d8de39bc076 pgstat: report in earlier with STATE_STARTING
    @@ Commit message
     
         2) pgstat_bestart_security() reports the SSL/GSS status of the
            connection.  Some backends don't call this at all; others call it
    -       twice, once after transport establishment and once after client
    -       authentication.
    +       after client authentication.
     
         3) pgstat_bestart_final() reports the user and database IDs, takes the
            entry out of STATE_STARTING, and reports the application_name.
    @@ src/backend/utils/activity/backend_status.c: pgstat_bestart(void)
     +}
     +
     +/*
    -+ * Fill in SSL and GSS information for the pgstat entry. This is separate from
    -+ * pgstat_bestart_initial() so that backends may call it multiple times as
    -+ * security details are filled in.
    ++ * Fill in SSL and GSS information for the pgstat entry.
     + *
     + * This should only be called from backends with a MyProcPort.
     + */
    @@ src/backend/utils/init/postinit.c: InitPostgres(const char *in_dbname, Oid dboid
     +	if (!bootstrap)
     +	{
     +		pgstat_bestart_initial();
    -+		if (MyProcPort)
    -+			pgstat_bestart_security();	/* fill in any SSL/GSS info too */
    -+
     +		INJECTION_POINT("init-pre-auth");
     +	}
     +
    @@ src/backend/utils/init/postinit.c: InitPostgres(const char *in_dbname, Oid dboid
      		InitializeSessionUserId(username, useroid, false);
      		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
     @@ src/backend/utils/init/postinit.c: InitPostgres(const char *in_dbname, Oid dboid,
    - 			InitializeSystemUser(MyClientConnectionInfo.authn_id,
    - 								 hba_authname(MyClientConnectionInfo.auth_method));
      		am_superuser = superuser();
    -+
    -+		/*
    -+		 * Authentication may have changed SSL/GSS details for the session, so
    -+		 * report it again.
    -+		 */
    -+		pgstat_bestart_security();
      	}
      
    ++	/* Report any SSL/GSS details for the session. */
    ++	if (MyProcPort != NULL)
    ++	{
    ++		Assert(!bootstrap);
    ++
    ++		pgstat_bestart_security();
    ++	}
    ++
      	/*
    + 	 * Binary upgrades only allowed super-user connections
    + 	 */
     @@ src/backend/utils/init/postinit.c: InitPostgres(const char *in_dbname, Oid dboid,
      		/* initialize client encoding */
      		InitializeClientEncoding();
    @@ src/test/authentication/t/007_pre_auth.pl (new)
     +$conn->quit();
     +
     +done_testing();
    -
    - ## src/test/ssl/Makefile ##
    -@@
    - #-------------------------------------------------------------------------
    - 
    - EXTRA_INSTALL = contrib/sslinfo
    -+EXTRA_INSTALL += src/test/modules/injection_points
    - 
    - subdir = src/test/ssl
    - top_builddir = ../../..
    - include $(top_builddir)/src/Makefile.global
    - 
    --export OPENSSL with_ssl
    -+export OPENSSL enable_injection_points with_ssl
    - 
    - # The sslfiles targets are separated into their own file due to interactions
    - # with settings in Makefile.global.
    -
    - ## src/test/ssl/meson.build ##
    -@@ src/test/ssl/meson.build: tests += {
    -   'bd': meson.current_build_dir(),
    -   'tap': {
    -     'env': {
    -+      'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
    -       'with_ssl': ssl_library,
    -       'OPENSSL': openssl.found() ? openssl.path() : '',
    -     },
    -
    - ## src/test/ssl/t/001_ssltests.pl ##
    -@@ src/test/ssl/t/001_ssltests.pl: use Config qw ( %Config );
    - use PostgreSQL::Test::Cluster;
    - use PostgreSQL::Test::Utils;
    - use Test::More;
    -+use Time::HiRes qw(usleep);
    - 
    - use FindBin;
    - use lib $FindBin::RealBin;
    -@@ src/test/ssl/t/001_ssltests.pl: $node->start;
    - my $result = $node->safe_psql('postgres', "SHOW ssl_library");
    - is($result, $ssl_server->ssl_library(), 'ssl_library parameter');
    - 
    -+my $injection_points_unavailable = '';
    -+if ($ENV{enable_injection_points} ne 'yes')
    -+{
    -+	$injection_points_unavailable =
    -+	  'Injection points not supported by this build';
    -+}
    -+elsif (!$node->check_extension('injection_points'))
    -+{
    -+	$injection_points_unavailable =
    -+	  'Extension injection_points not installed';
    -+}
    -+else
    -+{
    -+	# For ease of setup, make injection_points available for all new databases.
    -+	$node->safe_psql('template1', 'CREATE EXTENSION injection_points');
    -+}
    -+
    - $ssl_server->configure_test_server_for_ssl($node, $SERVERHOSTADDR,
    - 	$SERVERHOSTCIDR, 'trust');
    - 
    -@@ src/test/ssl/t/001_ssltests.pl: command_like(
    - 				^\d+,t,TLSv[\d.]+,[\w-]+,\d+,_null_,_null_,_null_\r?$}mx,
    - 	'pg_stat_ssl view without client certificate');
    - 
    -+# Test that pg_stat_ssl gets filled in early, prior to authentication. Requires
    -+# injection point support.
    -+SKIP:
    -+{
    -+	skip $injection_points_unavailable, 1 if $injection_points_unavailable;
    -+
    -+	# Connect to the server and inject a waitpoint.
    -+	my $psql =
    -+	  $node->background_psql('trustdb', connstr => "$common_connstr user=");
    -+	$psql->query_safe(
    -+		"SELECT injection_points_attach('init-pre-auth', 'wait')");
    -+
    -+	# From this point on, all new connections will hang during startup, just
    -+	# before authentication. Use the $psql connection handle for server
    -+	# interaction.
    -+	my $conn = $node->background_psql(
    -+		'trustdb',
    -+		connstr => $common_connstr,
    -+		wait => 0);
    -+
    -+	# Wait for the connection to show up.
    -+	my $pid;
    -+	while (1)
    -+	{
    -+		$pid = $psql->query(
    -+			"SELECT pid FROM pg_stat_activity WHERE state = 'starting' AND client_addr IS NOT NULL;");
    -+		last if $pid ne "";
    -+
    -+		usleep(100_000);
    -+	}
    -+
    -+	like(
    -+		$psql->query(
    -+			"SELECT ssl, version, cipher, bits FROM pg_stat_ssl WHERE pid = $pid"
    -+		),
    -+		qr/^t\|TLSv[\d.]+\|[\w-]+\|\d+$/,
    -+		'pg_stat_ssl view is updated prior to authentication');
    -+
    -+	# Detach the waitpoint and wait for the connection to complete.
    -+	$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
    -+	$conn->wait_connect();
    -+
    -+	$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
    -+	$psql->quit();
    -+	$conn->quit();
    -+}
    -+
    - # Test min/max SSL protocol versions.
    - $node->connect_ok(
    - 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require ssl_min_protocol_version=TLSv1.2 ssl_max_protocol_version=TLSv1.2",
2:  e734e46009f = 2:  fddaedf4280 Report external auth calls as wait events
3:  39c7d9ce42b = 3:  50d14d32699 squash! Report external auth calls as wait events

From d8de39bc0769cce3cb0c9d3d29cd528169d0a06f Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v9 1/3] pgstat: report in earlier with STATE_STARTING

Split pgstat_bestart() into three phases for better observability:

1) pgstat_bestart_initial() reports a 'starting' state while waiting for
   backend initialization and client authentication to complete.  Since
   we hold a transaction open for a good amount of that, and some
   authentication methods call out to external systems, having an early
   pg_stat_activity entry helps DBAs debug when things go badly wrong.

2) pgstat_bestart_security() reports the SSL/GSS status of the
   connection.  Some backends don't call this at all; others call it
   after client authentication.

3) pgstat_bestart_final() reports the user and database IDs, takes the
   entry out of STATE_STARTING, and reports the application_name.
   TODO: should the order of those last two be swapped?
---
 doc/src/sgml/monitoring.sgml                |   6 +
 src/backend/postmaster/auxprocess.c         |   3 +-
 src/backend/utils/activity/backend_status.c | 205 ++++++++++++--------
 src/backend/utils/adt/pgstatfuncs.c         |   3 +
 src/backend/utils/init/postinit.c           |  41 +++-
 src/include/utils/backend_status.h          |   5 +-
 src/test/authentication/Makefile            |   4 +
 src/test/authentication/meson.build         |   4 +
 src/test/authentication/t/007_pre_auth.pl   |  81 ++++++++
 9 files changed, 264 insertions(+), 88 deletions(-)
 create mode 100644 src/test/authentication/t/007_pre_auth.pl

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 9178f1d34ef..16646f560e8 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
diff --git a/src/backend/postmaster/auxprocess.c b/src/backend/postmaster/auxprocess.c
index ff366ceb0fc..4f6795f7265 100644
--- a/src/backend/postmaster/auxprocess.c
+++ b/src/backend/postmaster/auxprocess.c
@@ -78,7 +78,8 @@ AuxiliaryProcessMainCommon(void)
 
 	/* Initialize backend status information */
 	pgstat_beinit();
-	pgstat_bestart();
+	pgstat_bestart_initial();
+	pgstat_bestart_final();
 
 	/* register a before-shutdown callback for LWLock cleanup */
 	before_shmem_exit(ShutdownAuxiliaryProcess, 0);
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 5f68ef26adc..d84aea1a6b0 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -253,31 +253,18 @@ pgstat_beinit(void)
 	on_shmem_exit(pgstat_beshutdown_hook, 0);
 }
 
-
-/* ----------
- * pgstat_bestart() -
- *
- *	Initialize this backend's entry in the PgBackendStatus array.
- *	Called from InitPostgres.
- *
- *	Apart from auxiliary processes, MyDatabaseId, session userid, and
- *	application_name must already be set (hence, this cannot be combined
- *	with pgstat_beinit).  Note also that we must be inside a transaction
- *	if this isn't an aux process, as we may need to do encoding conversion
- *	on some strings.
- *----------
+/*
+ * Clears out a new pgstat entry, initializing it to suitable defaults and
+ * reporting STATE_STARTING. Backends should continue filling in any transport
+ * security details as needed with pgstat_bestart_security(), and must finally
+ * exit STATE_STARTING by calling pgstat_bestart_final(), once user and database
+ * IDs have been determined.
  */
 void
-pgstat_bestart(void)
+pgstat_bestart_initial(void)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
-#ifdef USE_SSL
-	PgBackendSSLStatus lsslstatus;
-#endif
-#ifdef ENABLE_GSS
-	PgBackendGSSStatus lgssstatus;
-#endif
 
 	/* pgstats state must be initialized from pgstat_beinit() */
 	Assert(vbeentry != NULL);
@@ -297,14 +284,6 @@ pgstat_bestart(void)
 		   unvolatize(PgBackendStatus *, vbeentry),
 		   sizeof(PgBackendStatus));
 
-	/* These structs can just start from zeroes each time, though */
-#ifdef USE_SSL
-	memset(&lsslstatus, 0, sizeof(lsslstatus));
-#endif
-#ifdef ENABLE_GSS
-	memset(&lgssstatus, 0, sizeof(lgssstatus));
-#endif
-
 	/*
 	 * Now fill in all the fields of lbeentry, except for strings that are
 	 * out-of-line data.  Those have to be handled separately, below.
@@ -315,15 +294,8 @@ pgstat_bestart(void)
 	lbeentry.st_activity_start_timestamp = 0;
 	lbeentry.st_state_start_timestamp = 0;
 	lbeentry.st_xact_start_timestamp = 0;
-	lbeentry.st_databaseid = MyDatabaseId;
-
-	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
-		lbeentry.st_userid = GetSessionUserId();
-	else
-		lbeentry.st_userid = InvalidOid;
+	lbeentry.st_databaseid = InvalidOid;
+	lbeentry.st_userid = InvalidOid;
 
 	/*
 	 * We may not have a MyProcPort (eg, if this is the autovacuum process).
@@ -336,46 +308,10 @@ pgstat_bestart(void)
 	else
 		MemSet(&lbeentry.st_clientaddr, 0, sizeof(lbeentry.st_clientaddr));
 
-#ifdef USE_SSL
-	if (MyProcPort && MyProcPort->ssl_in_use)
-	{
-		lbeentry.st_ssl = true;
-		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
-		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
-		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
-		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
-		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
-		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_ssl = false;
-	}
-#else
 	lbeentry.st_ssl = false;
-#endif
-
-#ifdef ENABLE_GSS
-	if (MyProcPort && MyProcPort->gss != NULL)
-	{
-		const char *princ = be_gssapi_get_princ(MyProcPort);
-
-		lbeentry.st_gss = true;
-		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
-		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
-		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
-		if (princ)
-			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_gss = false;
-	}
-#else
 	lbeentry.st_gss = false;
-#endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = STATE_STARTING;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
@@ -417,20 +353,137 @@ pgstat_bestart(void)
 	lbeentry.st_clienthostname[NAMEDATALEN - 1] = '\0';
 	lbeentry.st_activity_raw[pgstat_track_activity_query_size - 1] = '\0';
 
+	/* These structs can just start from zeroes each time */
 #ifdef USE_SSL
-	memcpy(lbeentry.st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+	memset(lbeentry.st_sslstatus, 0, sizeof(PgBackendSSLStatus));
 #endif
 #ifdef ENABLE_GSS
-	memcpy(lbeentry.st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+	memset(lbeentry.st_gssstatus, 0, sizeof(PgBackendGSSStatus));
 #endif
 
 	PGSTAT_END_WRITE_ACTIVITY(vbeentry);
+}
+
+/*
+ * Fill in SSL and GSS information for the pgstat entry.
+ *
+ * This should only be called from backends with a MyProcPort.
+ */
+void
+pgstat_bestart_security(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	bool		ssl = false;
+	bool		gss = false;
+#ifdef USE_SSL
+	PgBackendSSLStatus lsslstatus;
+	PgBackendSSLStatus *st_sslstatus;
+#endif
+#ifdef ENABLE_GSS
+	PgBackendGSSStatus lgssstatus;
+	PgBackendGSSStatus *st_gssstatus;
+#endif
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+	Assert(MyProcPort);			/* otherwise there's no point */
+
+#ifdef USE_SSL
+	st_sslstatus = beentry->st_sslstatus;
+	memset(&lsslstatus, 0, sizeof(lsslstatus));
+
+	if (MyProcPort->ssl_in_use)
+	{
+		ssl = true;
+		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
+		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
+		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
+		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
+		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
+		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
+	}
+#endif
+
+#ifdef ENABLE_GSS
+	st_gssstatus = beentry->st_gssstatus;
+	memset(&lgssstatus, 0, sizeof(lgssstatus));
+
+	if (MyProcPort->gss != NULL)
+	{
+		const char *princ = be_gssapi_get_princ(MyProcPort);
+
+		gss = true;
+		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
+		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
+		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
+		if (princ)
+			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
+	}
+#endif
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_ssl = ssl;
+	beentry->st_gss = gss;
+
+#ifdef USE_SSL
+	memcpy(st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+#endif
+#ifdef ENABLE_GSS
+	memcpy(st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+#endif
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/*
+ * Finalizes the startup pgstat entry by filling in the user/database IDs,
+ * clearing STATE_STARTING, and reporting the application_name.
+ *
+ * We must be inside a transaction if this isn't an aux process, as we may need
+ * to do encoding conversion.
+ */
+void
+pgstat_bestart_final(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	Oid			userid;
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+
+	/* We have userid for client-backends, wal-sender and bgworker processes */
+	if (MyBackendType == B_BACKEND
+		|| MyBackendType == B_WAL_SENDER
+		|| MyBackendType == B_BG_WORKER)
+		userid = GetSessionUserId();
+	else
+		userid = InvalidOid;
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_databaseid = MyDatabaseId;
+	beentry->st_userid = userid;
+	beentry->st_state = STATE_UNDEFINED;
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
 
 	/* Create the backend statistics entry */
 	if (pgstat_tracks_backend_bktype(MyBackendType))
 		pgstat_create_backend(MyProcNumber);
 
 	/* Update app name to current GUC setting */
+	/* TODO: ask the list: maybe do this before setting STATE_UNDEFINED? */
 	if (application_name)
 		pgstat_report_appname(application_name);
 }
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 68830db8633..bf972c9b67d 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -393,6 +393,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index 318600d6d02..6ed7024a614 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -59,6 +59,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -718,6 +719,20 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * This is a convenient time to sketch in a partial pgstat entry. That
+	 * way, if LWLocks or third-party authentication should happen to hang,
+	 * the DBA will still be able to see what's going on.
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_initial();
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -786,9 +801,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -809,8 +821,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* The autovacuum launcher is done here */
 	if (AmAutoVacuumLauncherProcess())
 	{
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		return;
 	}
@@ -875,6 +887,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	{
 		/* normal multiuser case */
 		Assert(MyProcPort != NULL);
+
 		PerformAuthentication(MyProcPort);
 		InitializeSessionUserId(username, useroid, false);
 		/* ensure that auth_method is actually valid, aka authn_id is not NULL */
@@ -884,6 +897,14 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		am_superuser = superuser();
 	}
 
+	/* Report any SSL/GSS details for the session. */
+	if (MyProcPort != NULL)
+	{
+		Assert(!bootstrap);
+
+		pgstat_bestart_security();
+	}
+
 	/*
 	 * Binary upgrades only allowed super-user connections
 	 */
@@ -953,8 +974,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		/* initialize client encoding */
 		InitializeClientEncoding();
 
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		/* close the transaction we started above */
 		CommitTransactionCommand();
@@ -997,7 +1018,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		 */
 		if (!bootstrap)
 		{
-			pgstat_bestart();
+			pgstat_bestart_final();
 			CommitTransactionCommand();
 		}
 		return;
@@ -1197,9 +1218,9 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	if ((flags & INIT_PG_LOAD_SESSION_LIBS) != 0)
 		process_session_preload_libraries();
 
-	/* report this backend in the PgBackendStatus array */
+	/* fill in the remainder of the PgBackendStatus array */
 	if (!bootstrap)
-		pgstat_bestart();
+		pgstat_bestart_final();
 
 	/* close the transaction we started above */
 	if (!bootstrap)
diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index d3d4ff6c5c9..1c9b4fe14d0 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,7 +310,9 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
-extern void pgstat_bestart(void);
+extern void pgstat_bestart_initial(void);
+extern void pgstat_bestart_security(void);
+extern void pgstat_bestart_final(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
 
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index c4022dc829b..8b5beced080 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,10 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+EXTRA_INSTALL = src/test/modules/injection_points
+
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index f6e48411c11..800b3a5ff40 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_pre_auth.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
new file mode 100644
index 00000000000..a638226dbaf
--- /dev/null
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -0,0 +1,81 @@
+
+# Copyright (c) 2021-2025, PostgreSQL Global Development Group
+
+# Tests for connection behavior prior to authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(100_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(100_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
-- 
2.34.1

From fddaedf4280fde31e7d495b312c24133ac0b198b Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v9 2/3] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 56 +++++++++++++++++--
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 27 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 16646f560e8..877163fb403 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 81e2f8184e3..bd8a2a098b3 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -39,6 +39,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -990,6 +991,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1001,6 +1003,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1212,6 +1215,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1221,6 +1225,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1286,6 +1292,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1295,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1392,11 +1400,13 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
@@ -1493,8 +1503,11 @@ pg_SSPI_make_upn(char *accountname,
 	 */
 
 	samname = psprintf("%s\\%s", domainname, accountname);
+
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						NULL, &upnamesize);
+	pgstat_report_wait_end();
 
 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
 		|| upnamesize == 0)
@@ -1509,8 +1522,10 @@ pg_SSPI_make_upn(char *accountname,
 	/* upnamesize includes the terminating NUL. */
 	upname = palloc(upnamesize);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						upname, &upnamesize);
+	pgstat_report_wait_end();
 
 	pfree(samname);
 	if (res)
@@ -2109,7 +2124,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2122,7 +2139,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2262,7 +2281,11 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 			}
 
 			/* Look up a list of LDAP server hosts and port numbers */
-			if (ldap_domain2hostlist(domain, &hostlist))
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
+			r = ldap_domain2hostlist(domain, &hostlist);
+			pgstat_report_wait_end();
+
+			if (r)
 			{
 				ereport(LOG,
 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
@@ -2356,11 +2379,15 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 
 	if (port->hba->ldaptls)
 	{
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
 #ifndef WIN32
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL);
 #else
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
 #endif
+		pgstat_report_wait_end();
+
+		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
 					(errmsg("could not start LDAP TLS session: %s",
@@ -2520,9 +2547,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2545,6 +2575,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2552,6 +2584,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2620,7 +2653,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -3069,8 +3104,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		return STATUS_ERROR;
 	}
 
-	if (sendto(sock, radius_buffer, packetlength, 0,
-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
+	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
+	r = sendto(sock, radius_buffer, packetlength, 0,
+			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
+	pgstat_report_wait_end();
+
+	if (r < 0)
 	{
 		ereport(LOG,
 				(errmsg("could not send RADIUS packet: %m")));
@@ -3118,7 +3157,10 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		FD_ZERO(&fdset);
 		FD_SET(sock, &fdset);
 
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
+		pgstat_report_wait_end();
+
 		if (r < 0)
 		{
 			if (errno == EINTR)
@@ -3151,8 +3193,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 
 		addrsize = sizeof(remoteaddr);
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
 								(struct sockaddr *) &remoteaddr, &addrsize);
+		pgstat_report_wait_end();
+
 		if (packetlength < 0)
 		{
 			ereport(LOG,
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d9b8f34a355..6cc3e1e7c7a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index e199f071628..a2852225614 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -162,6 +162,33 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
+RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
+RADIUS_WAIT	"Waiting for a RADIUS server to respond."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b8cb3e5a430..3d995a9e5be 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index 83228cfca29..b94930658b0 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -181,6 +181,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -189,7 +190,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

From 50d14d32699bfbad2e9c891b0017535f3d475249 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 8 Nov 2024 14:19:26 -0800
Subject: [PATCH v9 3/3] squash! Report external auth calls as wait events

Add a wait event around all calls to ldap_unbind(). (For the record, I
do not want to implement this in this way.)
---
 src/backend/libpq/auth.c                       | 18 ++++++++++++++++++
 .../utils/activity/wait_event_names.txt        |  9 +++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index bd8a2a098b3..34cc145dc5e 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -2373,7 +2373,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 				(errmsg("could not set LDAP protocol version: %s",
 						ldap_err2string(r)),
 				 errdetail_for_ldap(*ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SET_OPTION);
 		ldap_unbind(*ldap);
+		pgstat_report_wait_end();
 		return STATUS_ERROR;
 	}
 
@@ -2393,7 +2395,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 					(errmsg("could not start LDAP TLS session: %s",
 							ldap_err2string(r)),
 					 errdetail_for_ldap(*ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_START_TLS);
 			ldap_unbind(*ldap);
+			pgstat_report_wait_end();
 			return STATUS_ERROR;
 		}
 	}
@@ -2537,7 +2541,9 @@ CheckLDAPAuth(Port *port)
 			{
 				ereport(LOG,
 						(errmsg("invalid character in user name for LDAP authentication")));
+				pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_NAME_CHECK);
 				ldap_unbind(ldap);
+				pgstat_report_wait_end();
 				pfree(passwd);
 				return STATUS_ERROR;
 			}
@@ -2561,7 +2567,9 @@ CheckLDAPAuth(Port *port)
 							server_name,
 							ldap_err2string(r)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND_FOR_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			return STATUS_ERROR;
 		}
@@ -2594,7 +2602,9 @@ CheckLDAPAuth(Port *port)
 					 errdetail_for_ldap(ldap)));
 			if (search_message != NULL)
 				ldap_msgfree(search_message);
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			return STATUS_ERROR;
@@ -2616,7 +2626,9 @@ CheckLDAPAuth(Port *port)
 										  count,
 										  filter, server_name, count)));
 
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_COUNT_ENTRIES);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2635,7 +2647,9 @@ CheckLDAPAuth(Port *port)
 							filter, server_name,
 							ldap_err2string(error)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_GET_DN);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2663,7 +2677,9 @@ CheckLDAPAuth(Port *port)
 				(errmsg("LDAP login failed for user \"%s\" on server \"%s\": %s",
 						fulluser, server_name, ldap_err2string(r)),
 				 errdetail_for_ldap(ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND);
 		ldap_unbind(ldap);
+		pgstat_report_wait_end();
 		pfree(passwd);
 		pfree(fulluser);
 		return STATUS_ERROR;
@@ -2672,7 +2688,9 @@ CheckLDAPAuth(Port *port)
 	/* Save the original bind DN as the authenticated identity. */
 	set_authn_id(port, fulluser);
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_SUCCESS);
 	ldap_unbind(ldap);
+	pgstat_report_wait_end();
 	pfree(passwd);
 	pfree(fulluser);
 
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index a2852225614..f082756c294 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -177,6 +177,15 @@ LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory
 LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
 LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
 LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+LDAP_UNBIND_AFTER_BIND	"Waiting for an LDAP connection to be unbound after a simple bind failed."
+LDAP_UNBIND_AFTER_BIND_FOR_SEARCH	"Waiting for an LDAP connection to be unbound after a bind for search failed."
+LDAP_UNBIND_AFTER_COUNT_ENTRIES	"Waiting for an LDAP connection to be unbound after an entry count failed."
+LDAP_UNBIND_AFTER_GET_DN	"Waiting for an LDAP connection to be unbound after ldap_get_dn failed."
+LDAP_UNBIND_AFTER_NAME_CHECK	"Waiting for an LDAP connection to be unbound after a name check failed."
+LDAP_UNBIND_AFTER_SEARCH	"Waiting for an LDAP connection to be unbound after a bind+search failed."
+LDAP_UNBIND_AFTER_SET_OPTION	"Waiting for an LDAP connection to be unbound after ldap_set_option failed."
+LDAP_UNBIND_AFTER_START_TLS	"Waiting for an LDAP connection to be unbound after ldap_start_tls_s failed."
+LDAP_UNBIND_SUCCESS	"Waiting for a successful LDAP connection to be unbound."
 PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
 PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
 RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
-- 
2.34.1

From 301c381261c08efdcf2b129b8150db98eb101a97 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:54:58 -0700
Subject: [PATCH v10] pgstat: report in earlier with STATE_STARTING

Split pgstat_bestart() into three phases for better observability:

1) pgstat_bestart_initial() reports a 'starting' state while waiting for
   backend initialization and client authentication to complete.  Since
   we hold a transaction open for a good amount of that, and some
   authentication methods call out to external systems, having an early
   pg_stat_activity entry helps DBAs debug when things go badly wrong.

2) pgstat_bestart_security() reports the SSL/GSS status of the
   connection.  Some backends don't call this at all; others call it
   after client authentication.

3) pgstat_bestart_final() reports the user and database IDs, takes the
   entry out of STATE_STARTING, and reports the application_name.
   TODO: should the order of those last two be swapped?
---
 src/include/utils/backend_status.h          |   5 +-
 src/backend/postmaster/auxprocess.c         |   3 +-
 src/backend/utils/activity/backend_status.c | 215 +++++++++++++-------
 src/backend/utils/adt/pgstatfuncs.c         |   3 +
 src/backend/utils/init/postinit.c           |  40 +++-
 src/test/authentication/Makefile            |   4 +
 src/test/authentication/meson.build         |   4 +
 src/test/authentication/t/007_pre_auth.pl   |  81 ++++++++
 doc/src/sgml/monitoring.sgml                |   6 +
 9 files changed, 275 insertions(+), 86 deletions(-)
 create mode 100644 src/test/authentication/t/007_pre_auth.pl

diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h
index d3d4ff6c5c9a..1c9b4fe14d06 100644
--- a/src/include/utils/backend_status.h
+++ b/src/include/utils/backend_status.h
@@ -24,6 +24,7 @@
 typedef enum BackendState
 {
 	STATE_UNDEFINED,
+	STATE_STARTING,
 	STATE_IDLE,
 	STATE_RUNNING,
 	STATE_IDLEINTRANSACTION,
@@ -309,7 +310,9 @@ extern void BackendStatusShmemInit(void);
 
 /* Initialization functions */
 extern void pgstat_beinit(void);
-extern void pgstat_bestart(void);
+extern void pgstat_bestart_initial(void);
+extern void pgstat_bestart_security(void);
+extern void pgstat_bestart_final(void);
 
 extern void pgstat_clear_backend_activity_snapshot(void);
 
diff --git a/src/backend/postmaster/auxprocess.c b/src/backend/postmaster/auxprocess.c
index ff366ceb0fc7..4f6795f72650 100644
--- a/src/backend/postmaster/auxprocess.c
+++ b/src/backend/postmaster/auxprocess.c
@@ -78,7 +78,8 @@ AuxiliaryProcessMainCommon(void)
 
 	/* Initialize backend status information */
 	pgstat_beinit();
-	pgstat_bestart();
+	pgstat_bestart_initial();
+	pgstat_bestart_final();
 
 	/* register a before-shutdown callback for LWLock cleanup */
 	before_shmem_exit(ShutdownAuxiliaryProcess, 0);
diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c
index 5f68ef26adc6..1a4ca2b179ca 100644
--- a/src/backend/utils/activity/backend_status.c
+++ b/src/backend/utils/activity/backend_status.c
@@ -253,31 +253,23 @@ pgstat_beinit(void)
 	on_shmem_exit(pgstat_beshutdown_hook, 0);
 }
 
-
-/* ----------
- * pgstat_bestart() -
+/* ---------
+ * pgstat_bestart_initial() -
  *
- *	Initialize this backend's entry in the PgBackendStatus array.
- *	Called from InitPostgres.
+ * Initialize this backend's entry in the PgBackendStatus array.
+ * Called from InitPostgres() and AuxiliaryProcessMainCommon().
  *
- *	Apart from auxiliary processes, MyDatabaseId, session userid, and
- *	application_name must already be set (hence, this cannot be combined
- *	with pgstat_beinit).  Note also that we must be inside a transaction
- *	if this isn't an aux process, as we may need to do encoding conversion
- *	on some strings.
- *----------
+ * Clears out a new pgstat entry, initializing it to suitable defaults and
+ * reporting STATE_STARTING.  Backends should continue filling in any
+ * transport security details as needed with pgstat_bestart_security(), and
+ * must finally exit STATE_STARTING by calling pgstat_bestart_final().
+ * ---------
  */
 void
-pgstat_bestart(void)
+pgstat_bestart_initial(void)
 {
 	volatile PgBackendStatus *vbeentry = MyBEEntry;
 	PgBackendStatus lbeentry;
-#ifdef USE_SSL
-	PgBackendSSLStatus lsslstatus;
-#endif
-#ifdef ENABLE_GSS
-	PgBackendGSSStatus lgssstatus;
-#endif
 
 	/* pgstats state must be initialized from pgstat_beinit() */
 	Assert(vbeentry != NULL);
@@ -297,14 +289,6 @@ pgstat_bestart(void)
 		   unvolatize(PgBackendStatus *, vbeentry),
 		   sizeof(PgBackendStatus));
 
-	/* These structs can just start from zeroes each time, though */
-#ifdef USE_SSL
-	memset(&lsslstatus, 0, sizeof(lsslstatus));
-#endif
-#ifdef ENABLE_GSS
-	memset(&lgssstatus, 0, sizeof(lgssstatus));
-#endif
-
 	/*
 	 * Now fill in all the fields of lbeentry, except for strings that are
 	 * out-of-line data.  Those have to be handled separately, below.
@@ -315,15 +299,8 @@ pgstat_bestart(void)
 	lbeentry.st_activity_start_timestamp = 0;
 	lbeentry.st_state_start_timestamp = 0;
 	lbeentry.st_xact_start_timestamp = 0;
-	lbeentry.st_databaseid = MyDatabaseId;
-
-	/* We have userid for client-backends, wal-sender and bgworker processes */
-	if (lbeentry.st_backendType == B_BACKEND
-		|| lbeentry.st_backendType == B_WAL_SENDER
-		|| lbeentry.st_backendType == B_BG_WORKER)
-		lbeentry.st_userid = GetSessionUserId();
-	else
-		lbeentry.st_userid = InvalidOid;
+	lbeentry.st_databaseid = InvalidOid;
+	lbeentry.st_userid = InvalidOid;
 
 	/*
 	 * We may not have a MyProcPort (eg, if this is the autovacuum process).
@@ -336,46 +313,10 @@ pgstat_bestart(void)
 	else
 		MemSet(&lbeentry.st_clientaddr, 0, sizeof(lbeentry.st_clientaddr));
 
-#ifdef USE_SSL
-	if (MyProcPort && MyProcPort->ssl_in_use)
-	{
-		lbeentry.st_ssl = true;
-		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
-		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
-		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
-		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
-		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
-		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_ssl = false;
-	}
-#else
 	lbeentry.st_ssl = false;
-#endif
-
-#ifdef ENABLE_GSS
-	if (MyProcPort && MyProcPort->gss != NULL)
-	{
-		const char *princ = be_gssapi_get_princ(MyProcPort);
-
-		lbeentry.st_gss = true;
-		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
-		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
-		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
-		if (princ)
-			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
-	}
-	else
-	{
-		lbeentry.st_gss = false;
-	}
-#else
 	lbeentry.st_gss = false;
-#endif
 
-	lbeentry.st_state = STATE_UNDEFINED;
+	lbeentry.st_state = STATE_STARTING;
 	lbeentry.st_progress_command = PROGRESS_COMMAND_INVALID;
 	lbeentry.st_progress_command_target = InvalidOid;
 	lbeentry.st_query_id = UINT64CONST(0);
@@ -417,20 +358,146 @@ pgstat_bestart(void)
 	lbeentry.st_clienthostname[NAMEDATALEN - 1] = '\0';
 	lbeentry.st_activity_raw[pgstat_track_activity_query_size - 1] = '\0';
 
+	/* These structs can just start from zeroes each time */
 #ifdef USE_SSL
-	memcpy(lbeentry.st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+	memset(lbeentry.st_sslstatus, 0, sizeof(PgBackendSSLStatus));
 #endif
 #ifdef ENABLE_GSS
-	memcpy(lbeentry.st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+	memset(lbeentry.st_gssstatus, 0, sizeof(PgBackendGSSStatus));
 #endif
 
 	PGSTAT_END_WRITE_ACTIVITY(vbeentry);
+}
+
+/* --------
+ * pgstat_bestart_security() -
+ *
+ * Fill in SSL and GSS information for the pgstat entry.  This is the second
+ * optional step taken when filling a backend's entry, not required for
+ * auxiliary processes.
+ *
+ * This should only be called from backends with a MyProcPort.
+ * --------
+ */
+void
+pgstat_bestart_security(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	bool		ssl = false;
+	bool		gss = false;
+#ifdef USE_SSL
+	PgBackendSSLStatus lsslstatus;
+	PgBackendSSLStatus *st_sslstatus;
+#endif
+#ifdef ENABLE_GSS
+	PgBackendGSSStatus lgssstatus;
+	PgBackendGSSStatus *st_gssstatus;
+#endif
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+	Assert(MyProcPort);			/* otherwise there's no point */
+
+#ifdef USE_SSL
+	st_sslstatus = beentry->st_sslstatus;
+	memset(&lsslstatus, 0, sizeof(lsslstatus));
+
+	if (MyProcPort->ssl_in_use)
+	{
+		ssl = true;
+		lsslstatus.ssl_bits = be_tls_get_cipher_bits(MyProcPort);
+		strlcpy(lsslstatus.ssl_version, be_tls_get_version(MyProcPort), NAMEDATALEN);
+		strlcpy(lsslstatus.ssl_cipher, be_tls_get_cipher(MyProcPort), NAMEDATALEN);
+		be_tls_get_peer_subject_name(MyProcPort, lsslstatus.ssl_client_dn, NAMEDATALEN);
+		be_tls_get_peer_serial(MyProcPort, lsslstatus.ssl_client_serial, NAMEDATALEN);
+		be_tls_get_peer_issuer_name(MyProcPort, lsslstatus.ssl_issuer_dn, NAMEDATALEN);
+	}
+#endif
+
+#ifdef ENABLE_GSS
+	st_gssstatus = beentry->st_gssstatus;
+	memset(&lgssstatus, 0, sizeof(lgssstatus));
+
+	if (MyProcPort->gss != NULL)
+	{
+		const char *princ = be_gssapi_get_princ(MyProcPort);
+
+		gss = true;
+		lgssstatus.gss_auth = be_gssapi_get_auth(MyProcPort);
+		lgssstatus.gss_enc = be_gssapi_get_enc(MyProcPort);
+		lgssstatus.gss_delegation = be_gssapi_get_delegation(MyProcPort);
+		if (princ)
+			strlcpy(lgssstatus.gss_princ, princ, NAMEDATALEN);
+	}
+#endif
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_ssl = ssl;
+	beentry->st_gss = gss;
+
+#ifdef USE_SSL
+	memcpy(st_sslstatus, &lsslstatus, sizeof(PgBackendSSLStatus));
+#endif
+#ifdef ENABLE_GSS
+	memcpy(st_gssstatus, &lgssstatus, sizeof(PgBackendGSSStatus));
+#endif
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/* --------
+ * pgstat_bestart_final() -
+ *
+ * Finalizes the state of this backend's entry entry by filling in the
+ * user and database IDs, clearing STATE_STARTING, and reporting the
+ * application_name.
+ *
+ * We must be inside a transaction if this is not an auxiliary process, as
+ * we may need to do encoding conversion.
+ * --------
+ */
+void
+pgstat_bestart_final(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	Oid			userid;
+
+	/* pgstats state must be initialized from pgstat_beinit() */
+	Assert(beentry != NULL);
+
+	/* We have userid for client-backends, wal-sender and bgworker processes */
+	if (MyBackendType == B_BACKEND
+		|| MyBackendType == B_WAL_SENDER
+		|| MyBackendType == B_BG_WORKER)
+		userid = GetSessionUserId();
+	else
+		userid = InvalidOid;
+
+	/*
+	 * Update my status entry, following the protocol of bumping
+	 * st_changecount before and after.  We use a volatile pointer here to
+	 * ensure the compiler doesn't try to get cute.
+	 */
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	beentry->st_databaseid = MyDatabaseId;
+	beentry->st_userid = userid;
+	beentry->st_state = STATE_UNDEFINED;
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
 
 	/* Create the backend statistics entry */
 	if (pgstat_tracks_backend_bktype(MyBackendType))
 		pgstat_create_backend(MyProcNumber);
 
 	/* Update app name to current GUC setting */
+	/* TODO: ask the list: maybe do this before setting STATE_UNDEFINED? */
 	if (application_name)
 		pgstat_report_appname(application_name);
 }
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 0212d8d5906b..9172e1cb9d23 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -393,6 +393,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS)
 
 			switch (beentry->st_state)
 			{
+				case STATE_STARTING:
+					values[4] = CStringGetTextDatum("starting");
+					break;
 				case STATE_IDLE:
 					values[4] = CStringGetTextDatum("idle");
 					break;
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index 318600d6d02e..b428a59bdd26 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -59,6 +59,7 @@
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/guc_hooks.h"
+#include "utils/injection_point.h"
 #include "utils/memutils.h"
 #include "utils/pg_locale.h"
 #include "utils/portal.h"
@@ -718,6 +719,20 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	 */
 	InitProcessPhase2();
 
+	/* Initialize status reporting */
+	pgstat_beinit();
+
+	/*
+	 * And initialize an entry in the PgBackendStatus array.  That way, if
+	 * LWLocks or third-party authentication should happen to hang, it is
+	 * possible to retrieve some information about what is going on.
+	 */
+	if (!bootstrap)
+	{
+		pgstat_bestart_initial();
+		INJECTION_POINT("init-pre-auth");
+	}
+
 	/*
 	 * Initialize my entry in the shared-invalidation manager's array of
 	 * per-backend data.
@@ -786,9 +801,6 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* Initialize portal manager */
 	EnablePortalManager();
 
-	/* Initialize status reporting */
-	pgstat_beinit();
-
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.
@@ -809,8 +821,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	/* The autovacuum launcher is done here */
 	if (AmAutoVacuumLauncherProcess())
 	{
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of this entry in the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		return;
 	}
@@ -884,6 +896,14 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		am_superuser = superuser();
 	}
 
+	/* Report any SSL/GSS details for the session. */
+	if (MyProcPort != NULL)
+	{
+		Assert(!bootstrap);
+
+		pgstat_bestart_security();
+	}
+
 	/*
 	 * Binary upgrades only allowed super-user connections
 	 */
@@ -953,8 +973,8 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		/* initialize client encoding */
 		InitializeClientEncoding();
 
-		/* report this backend in the PgBackendStatus array */
-		pgstat_bestart();
+		/* fill in the remainder of this entry in the PgBackendStatus array */
+		pgstat_bestart_final();
 
 		/* close the transaction we started above */
 		CommitTransactionCommand();
@@ -997,7 +1017,7 @@ InitPostgres(const char *in_dbname, Oid dboid,
 		 */
 		if (!bootstrap)
 		{
-			pgstat_bestart();
+			pgstat_bestart_final();
 			CommitTransactionCommand();
 		}
 		return;
@@ -1197,9 +1217,9 @@ InitPostgres(const char *in_dbname, Oid dboid,
 	if ((flags & INIT_PG_LOAD_SESSION_LIBS) != 0)
 		process_session_preload_libraries();
 
-	/* report this backend in the PgBackendStatus array */
+	/* fill in the remainder of this entry in the PgBackendStatus array */
 	if (!bootstrap)
-		pgstat_bestart();
+		pgstat_bestart_final();
 
 	/* close the transaction we started above */
 	if (!bootstrap)
diff --git a/src/test/authentication/Makefile b/src/test/authentication/Makefile
index c4022dc829b6..8b5beced0806 100644
--- a/src/test/authentication/Makefile
+++ b/src/test/authentication/Makefile
@@ -13,6 +13,10 @@ subdir = src/test/authentication
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+EXTRA_INSTALL = src/test/modules/injection_points
+
+export enable_injection_points
+
 check:
 	$(prove_check)
 
diff --git a/src/test/authentication/meson.build b/src/test/authentication/meson.build
index f6e48411c116..800b3a5ff40f 100644
--- a/src/test/authentication/meson.build
+++ b/src/test/authentication/meson.build
@@ -5,6 +5,9 @@ tests += {
   'sd': meson.current_source_dir(),
   'bd': meson.current_build_dir(),
   'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
     'tests': [
       't/001_password.pl',
       't/002_saslprep.pl',
@@ -12,6 +15,7 @@ tests += {
       't/004_file_inclusion.pl',
       't/005_sspi.pl',
       't/006_login_trigger.pl',
+      't/007_pre_auth.pl',
     ],
   },
 }
diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
new file mode 100644
index 000000000000..a638226dbaf1
--- /dev/null
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -0,0 +1,81 @@
+
+# Copyright (c) 2021-2025, PostgreSQL Global Development Group
+
+# Tests for connection behavior prior to authentication.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Time::HiRes qw(usleep);
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q[
+log_connections = on
+]);
+
+$node->start;
+
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points');
+
+# Connect to the server and inject a waitpoint.
+my $psql = $node->background_psql('postgres');
+$psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
+
+# From this point on, all new connections will hang during startup, just before
+# authentication. Use the $psql connection handle for server interaction.
+my $conn = $node->background_psql('postgres', wait => 0);
+
+# Wait for the connection to show up.
+my $pid;
+while (1)
+{
+	$pid = $psql->query(
+		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+	last if $pid ne "";
+
+	usleep(100_000);
+}
+
+note "backend $pid is authenticating";
+ok(1, 'authenticating connections are recorded in pg_stat_activity');
+
+# Detach the waitpoint and wait for the connection to complete.
+$psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
+$conn->wait_connect();
+
+# Make sure the pgstat entry is updated eventually.
+while (1)
+{
+	my $state =
+	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
+	last if $state eq "idle";
+
+	note "state for backend $pid is '$state'; waiting for 'idle'...";
+	usleep(100_000);
+}
+
+ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+
+$psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
+$psql->quit();
+$conn->quit();
+
+done_testing();
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 9178f1d34efd..16646f560e8d 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -899,6 +899,12 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        Current overall state of this backend.
        Possible values are:
        <itemizedlist>
+        <listitem>
+         <para>
+          <literal>starting</literal>: The backend is in initial startup. Client
+          authentication is performed during this phase.
+         </para>
+        </listitem>
         <listitem>
         <para>
           <literal>active</literal>: The backend is executing a query.
-- 
2.47.2

From 75de9ca51a61f0860a11259f3624b2eb12a4cb01 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v10 1/2] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 src/include/utils/wait_event.h                |  1 +
 src/backend/libpq/auth.c                      | 56 +++++++++++++++++--
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 27 +++++++++
 src/test/regress/expected/sysviews.out        |  3 +-
 doc/src/sgml/monitoring.sgml                  |  8 +++
 6 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b8cb3e5a4309..3d995a9e5be7 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 81e2f8184e30..bd8a2a098b39 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -39,6 +39,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -990,6 +991,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1001,6 +1003,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1212,6 +1215,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1221,6 +1225,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1286,6 +1292,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1295,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1392,11 +1400,13 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
@@ -1493,8 +1503,11 @@ pg_SSPI_make_upn(char *accountname,
 	 */
 
 	samname = psprintf("%s\\%s", domainname, accountname);
+
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						NULL, &upnamesize);
+	pgstat_report_wait_end();
 
 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
 		|| upnamesize == 0)
@@ -1509,8 +1522,10 @@ pg_SSPI_make_upn(char *accountname,
 	/* upnamesize includes the terminating NUL. */
 	upname = palloc(upnamesize);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						upname, &upnamesize);
+	pgstat_report_wait_end();
 
 	pfree(samname);
 	if (res)
@@ -2109,7 +2124,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2122,7 +2139,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2262,7 +2281,11 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 			}
 
 			/* Look up a list of LDAP server hosts and port numbers */
-			if (ldap_domain2hostlist(domain, &hostlist))
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
+			r = ldap_domain2hostlist(domain, &hostlist);
+			pgstat_report_wait_end();
+
+			if (r)
 			{
 				ereport(LOG,
 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
@@ -2356,11 +2379,15 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 
 	if (port->hba->ldaptls)
 	{
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
 #ifndef WIN32
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL);
 #else
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
 #endif
+		pgstat_report_wait_end();
+
+		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
 					(errmsg("could not start LDAP TLS session: %s",
@@ -2520,9 +2547,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2545,6 +2575,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2552,6 +2584,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2620,7 +2653,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -3069,8 +3104,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		return STATUS_ERROR;
 	}
 
-	if (sendto(sock, radius_buffer, packetlength, 0,
-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
+	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
+	r = sendto(sock, radius_buffer, packetlength, 0,
+			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
+	pgstat_report_wait_end();
+
+	if (r < 0)
 	{
 		ereport(LOG,
 				(errmsg("could not send RADIUS packet: %m")));
@@ -3118,7 +3157,10 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		FD_ZERO(&fdset);
 		FD_SET(sock, &fdset);
 
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
+		pgstat_report_wait_end();
+
 		if (r < 0)
 		{
 			if (errno == EINTR)
@@ -3151,8 +3193,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 
 		addrsize = sizeof(remoteaddr);
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
 								(struct sockaddr *) &remoteaddr, &addrsize);
+		pgstat_report_wait_end();
+
 		if (packetlength < 0)
 		{
 			ereport(LOG,
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d9b8f34a3559..6cc3e1e7c7a0 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index e199f0716289..a28522256140 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -162,6 +162,33 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
+RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
+RADIUS_WAIT	"Waiting for a RADIUS server to respond."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index 83228cfca293..b94930658b0e 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -181,6 +181,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -189,7 +190,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 16646f560e8d..877163fb4036 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
-- 
2.47.2

From a7e18bff1aa45e1e6602581cca4d5a0efe8e5d07 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 8 Nov 2024 14:19:26 -0800
Subject: [PATCH v10 2/2] squash! Report external auth calls as wait events

Add a wait event around all calls to ldap_unbind(). (For the record, I
do not want to implement this in this way.)
---
 src/backend/libpq/auth.c                       | 18 ++++++++++++++++++
 .../utils/activity/wait_event_names.txt        |  9 +++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index bd8a2a098b39..34cc145dc5e8 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -2373,7 +2373,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 				(errmsg("could not set LDAP protocol version: %s",
 						ldap_err2string(r)),
 				 errdetail_for_ldap(*ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SET_OPTION);
 		ldap_unbind(*ldap);
+		pgstat_report_wait_end();
 		return STATUS_ERROR;
 	}
 
@@ -2393,7 +2395,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 					(errmsg("could not start LDAP TLS session: %s",
 							ldap_err2string(r)),
 					 errdetail_for_ldap(*ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_START_TLS);
 			ldap_unbind(*ldap);
+			pgstat_report_wait_end();
 			return STATUS_ERROR;
 		}
 	}
@@ -2537,7 +2541,9 @@ CheckLDAPAuth(Port *port)
 			{
 				ereport(LOG,
 						(errmsg("invalid character in user name for LDAP authentication")));
+				pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_NAME_CHECK);
 				ldap_unbind(ldap);
+				pgstat_report_wait_end();
 				pfree(passwd);
 				return STATUS_ERROR;
 			}
@@ -2561,7 +2567,9 @@ CheckLDAPAuth(Port *port)
 							server_name,
 							ldap_err2string(r)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND_FOR_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			return STATUS_ERROR;
 		}
@@ -2594,7 +2602,9 @@ CheckLDAPAuth(Port *port)
 					 errdetail_for_ldap(ldap)));
 			if (search_message != NULL)
 				ldap_msgfree(search_message);
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			return STATUS_ERROR;
@@ -2616,7 +2626,9 @@ CheckLDAPAuth(Port *port)
 										  count,
 										  filter, server_name, count)));
 
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_COUNT_ENTRIES);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2635,7 +2647,9 @@ CheckLDAPAuth(Port *port)
 							filter, server_name,
 							ldap_err2string(error)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_GET_DN);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2663,7 +2677,9 @@ CheckLDAPAuth(Port *port)
 				(errmsg("LDAP login failed for user \"%s\" on server \"%s\": %s",
 						fulluser, server_name, ldap_err2string(r)),
 				 errdetail_for_ldap(ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND);
 		ldap_unbind(ldap);
+		pgstat_report_wait_end();
 		pfree(passwd);
 		pfree(fulluser);
 		return STATUS_ERROR;
@@ -2672,7 +2688,9 @@ CheckLDAPAuth(Port *port)
 	/* Save the original bind DN as the authenticated identity. */
 	set_authn_id(port, fulluser);
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_SUCCESS);
 	ldap_unbind(ldap);
+	pgstat_report_wait_end();
 	pfree(passwd);
 	pfree(fulluser);
 
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index a28522256140..f082756c294e 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -177,6 +177,15 @@ LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory
 LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
 LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
 LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+LDAP_UNBIND_AFTER_BIND	"Waiting for an LDAP connection to be unbound after a simple bind failed."
+LDAP_UNBIND_AFTER_BIND_FOR_SEARCH	"Waiting for an LDAP connection to be unbound after a bind for search failed."
+LDAP_UNBIND_AFTER_COUNT_ENTRIES	"Waiting for an LDAP connection to be unbound after an entry count failed."
+LDAP_UNBIND_AFTER_GET_DN	"Waiting for an LDAP connection to be unbound after ldap_get_dn failed."
+LDAP_UNBIND_AFTER_NAME_CHECK	"Waiting for an LDAP connection to be unbound after a name check failed."
+LDAP_UNBIND_AFTER_SEARCH	"Waiting for an LDAP connection to be unbound after a bind+search failed."
+LDAP_UNBIND_AFTER_SET_OPTION	"Waiting for an LDAP connection to be unbound after ldap_set_option failed."
+LDAP_UNBIND_AFTER_START_TLS	"Waiting for an LDAP connection to be unbound after ldap_start_tls_s failed."
+LDAP_UNBIND_SUCCESS	"Waiting for a successful LDAP connection to be unbound."
 PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
 PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
 RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
-- 
2.47.2

From d5c976dc995bf09c4868e0b07d4652dabcaec888 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Wed, 5 Mar 2025 13:30:43 +0900
Subject: [PATCH] Fix race condition in pre-auth test

---
 src/test/authentication/t/007_pre_auth.pl | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
index a638226dbaf1..90aaea4b5a64 100644
--- a/src/test/authentication/t/007_pre_auth.pl
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -43,12 +43,14 @@ $psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
 # authentication. Use the $psql connection handle for server interaction.
 my $conn = $node->background_psql('postgres', wait => 0);
 
-# Wait for the connection to show up.
+# Wait for the connection to show up in pg_stat_activity, with the wait_event
+# of the injection point.
 my $pid;
 while (1)
 {
 	$pid = $psql->query(
-		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+		qq{SELECT pid FROM pg_stat_activity
+  WHERE state = 'starting' and wait_event = 'init-pre-auth';});
 	last if $pid ne "";
 
 	usleep(100_000);
-- 
2.47.2

From efc9fc3b3993601e9611131f229fbcaf4daa46f1 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Wed, 5 Mar 2025 13:30:43 +0900
Subject: [PATCH 1/2] Fix race condition in pre-auth test

---
 src/test/authentication/t/007_pre_auth.pl | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
index a638226dbaf..90aaea4b5a6 100644
--- a/src/test/authentication/t/007_pre_auth.pl
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -43,12 +43,14 @@ $psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
 # authentication. Use the $psql connection handle for server interaction.
 my $conn = $node->background_psql('postgres', wait => 0);
 
-# Wait for the connection to show up.
+# Wait for the connection to show up in pg_stat_activity, with the wait_event
+# of the injection point.
 my $pid;
 while (1)
 {
 	$pid = $psql->query(
-		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
+		qq{SELECT pid FROM pg_stat_activity
+  WHERE state = 'starting' and wait_event = 'init-pre-auth';});
 	last if $pid ne "";
 
 	usleep(100_000);
-- 
2.34.1

From 6ef54e90ec5eeb92ec8bd857f6d6f35578f058b9 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Wed, 5 Mar 2025 08:13:22 -0800
Subject: [PATCH 2/2] squash! Fix race condition in pre-auth test

Wait only for client backends.
---
 src/test/authentication/t/007_pre_auth.pl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
index 90aaea4b5a6..12e40dc722c 100644
--- a/src/test/authentication/t/007_pre_auth.pl
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -50,7 +50,9 @@ while (1)
 {
 	$pid = $psql->query(
 		qq{SELECT pid FROM pg_stat_activity
-  WHERE state = 'starting' and wait_event = 'init-pre-auth';});
+  WHERE backend_type = 'client backend'
+    AND state = 'starting'
+    AND wait_event = 'init-pre-auth';});
 	last if $pid ne "";
 
 	usleep(100_000);
-- 
2.34.1

From 8caea1d434aa1cbd6f1da777b89dd4895a88b44f Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Thu, 6 Mar 2025 13:06:05 +0900
Subject: [PATCH v2] Fix race condition in pre-auth test

---
 src/test/authentication/t/007_pre_auth.pl     | 36 ++++++---------
 .../perl/PostgreSQL/Test/BackgroundPsql.pm    | 44 +++++++++++++++++++
 2 files changed, 57 insertions(+), 23 deletions(-)

diff --git a/src/test/authentication/t/007_pre_auth.pl b/src/test/authentication/t/007_pre_auth.pl
index a638226dbaf1..fb4241cac4ab 100644
--- a/src/test/authentication/t/007_pre_auth.pl
+++ b/src/test/authentication/t/007_pre_auth.pl
@@ -43,36 +43,26 @@ $psql->query_safe("SELECT injection_points_attach('init-pre-auth', 'wait')");
 # authentication. Use the $psql connection handle for server interaction.
 my $conn = $node->background_psql('postgres', wait => 0);
 
-# Wait for the connection to show up.
-my $pid;
-while (1)
-{
-	$pid = $psql->query(
-		"SELECT pid FROM pg_stat_activity WHERE state = 'starting';");
-	last if $pid ne "";
+# Wait for the connection to show up in pg_stat_activity, with the wait_event
+# of the injection point.
+my $res = $psql->poll_query_until(
+	qq{SELECT count(pid) > 0 FROM pg_stat_activity
+  WHERE state = 'starting' and wait_event = 'init-pre-auth';});
+ok($res, 'authenticating connections are recorded in pg_stat_activity');
 
-	usleep(100_000);
-}
-
-note "backend $pid is authenticating";
-ok(1, 'authenticating connections are recorded in pg_stat_activity');
+# Get the PID of the backend waiting, for the next checks.
+my $pid = $psql->query(
+	qq{SELECT pid FROM pg_stat_activity
+  WHERE state = 'starting' and wait_event = 'init-pre-auth';});
 
 # Detach the waitpoint and wait for the connection to complete.
 $psql->query_safe("SELECT injection_points_wakeup('init-pre-auth');");
 $conn->wait_connect();
 
 # Make sure the pgstat entry is updated eventually.
-while (1)
-{
-	my $state =
-	  $psql->query("SELECT state FROM pg_stat_activity WHERE pid = $pid;");
-	last if $state eq "idle";
-
-	note "state for backend $pid is '$state'; waiting for 'idle'...";
-	usleep(100_000);
-}
-
-ok(1, 'authenticated connections reach idle state in pg_stat_activity');
+$res = $psql->poll_query_until(
+	qq{SELECT state FROM pg_stat_activity WHERE pid = $pid;}, 'idle');
+ok($res, 'authenticated connections reach idle state in pg_stat_activity');
 
 $psql->query_safe("SELECT injection_points_detach('init-pre-auth');");
 $psql->quit();
diff --git a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
index c611a61cf4e6..83ecf8b3e720 100644
--- a/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
+++ b/src/test/perl/PostgreSQL/Test/BackgroundPsql.pm
@@ -61,6 +61,7 @@ use Config;
 use IPC::Run;
 use PostgreSQL::Test::Utils qw(pump_until);
 use Test::More;
+use Time::HiRes qw(usleep);
 
 =pod
 
@@ -172,6 +173,49 @@ sub wait_connect
 	die "psql startup timed out" if $self->{timeout}->is_expired;
 }
 
+
+=pod
+
+=item $session->poll_query_until($query [, $expected ])
+
+Run B<$query> repeatedly, until it returns the B<$expected> result
+('t', or SQL boolean true, by default).
+Continues polling if B<query> returns an error result.
+
+Times out after $PostgreSQL::Test::Utils::timeout_default seconds.
+
+Returns 1 if successful, 0 if timed out.
+
+=cut
+
+sub poll_query_until
+{
+	my ($self, $query, $expected) = @_;
+
+	$expected = 't' unless defined($expected);    # default value
+
+	my $max_attempts = 10 * $PostgreSQL::Test::Utils::timeout_default;
+	my $attempts = 0;
+
+	while ($attempts < $max_attempts)
+	{
+		my $ret = $self->query($query);
+
+		if ($ret eq $expected)
+		{
+			return 1;
+		}
+
+		# Wait 0.1 second before retrying.
+		usleep(100_000);
+		$attempts++;
+	}
+
+	# Give up.  The output of the last attempt is logged by query(),
+	# so no need to do anything here.
+	return 0;
+}
+
 =pod
 
 =item $session->quit
-- 
2.47.2

From 0dc8e12758a1e7bdde7e5e4e60570b8b4f75d240 Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 3 May 2024 15:58:23 -0700
Subject: [PATCH v11 1/2] Report external auth calls as wait events

Introduce a new "Auth" wait class for various external authentication
systems, to make it obvious what's going wrong if one of those systems
hangs. Each new wait event is unique in order to more easily pinpoint
problematic locations in the code.

Discussion: https://postgr.es/m/CAOYmi%2B%3D60deN20WDyCoHCiecgivJxr%3D98s7s7-C8SkXwrCfHXg%40mail.gmail.com
---
 doc/src/sgml/monitoring.sgml                  |  8 +++
 src/backend/libpq/auth.c                      | 56 +++++++++++++++++--
 src/backend/utils/activity/wait_event.c       | 11 ++++
 .../utils/activity/wait_event_names.txt       | 27 +++++++++
 src/include/utils/wait_event.h                |  1 +
 src/test/regress/expected/sysviews.out        |  3 +-
 6 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 16646f560e8..877163fb403 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1045,6 +1045,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
        see <xref linkend="wait-event-activity-table"/>.
       </entry>
      </row>
+     <row>
+      <entry><literal>Auth</literal></entry>
+      <entry>The server process is waiting for an external system to
+       authenticate and/or authorize the client connection.
+       <literal>wait_event</literal> will identify the specific wait point;
+       see <xref linkend="wait-event-auth-table"/>.
+      </entry>
+     </row>
      <row>
       <entry><literal>BufferPin</literal></entry>
       <entry>The server process is waiting for exclusive access to
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index 81e2f8184e3..bd8a2a098b3 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -39,6 +39,7 @@
 #include "replication/walsender.h"
 #include "storage/ipc.h"
 #include "utils/memutils.h"
+#include "utils/wait_event.h"
 
 /*----------------------------------------------------------------
  * Global authentication functions
@@ -990,6 +991,7 @@ pg_GSS_recvauth(Port *port)
 		elog(DEBUG4, "processing received GSS token of length %u",
 			 (unsigned int) gbuf.length);
 
+		pgstat_report_wait_start(WAIT_EVENT_GSSAPI_ACCEPT_SEC_CONTEXT);
 		maj_stat = gss_accept_sec_context(&min_stat,
 										  &port->gss->ctx,
 										  port->gss->cred,
@@ -1001,6 +1003,7 @@ pg_GSS_recvauth(Port *port)
 										  &gflags,
 										  NULL,
 										  pg_gss_accept_delegation ? &delegated_creds : NULL);
+		pgstat_report_wait_end();
 
 		/* gbuf no longer used */
 		pfree(buf.data);
@@ -1212,6 +1215,7 @@ pg_SSPI_recvauth(Port *port)
 	/*
 	 * Acquire a handle to the server credentials.
 	 */
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_ACQUIRE_CREDENTIALS_HANDLE);
 	r = AcquireCredentialsHandle(NULL,
 								 "negotiate",
 								 SECPKG_CRED_INBOUND,
@@ -1221,6 +1225,8 @@ pg_SSPI_recvauth(Port *port)
 								 NULL,
 								 &sspicred,
 								 &expiry);
+	pgstat_report_wait_end();
+
 	if (r != SEC_E_OK)
 		pg_SSPI_error(ERROR, _("could not acquire SSPI credentials"), r);
 
@@ -1286,6 +1292,7 @@ pg_SSPI_recvauth(Port *port)
 		elog(DEBUG4, "processing received SSPI token of length %u",
 			 (unsigned int) buf.len);
 
+		pgstat_report_wait_start(WAIT_EVENT_SSPI_ACCEPT_SECURITY_CONTEXT);
 		r = AcceptSecurityContext(&sspicred,
 								  sspictx,
 								  &inbuf,
@@ -1295,6 +1302,7 @@ pg_SSPI_recvauth(Port *port)
 								  &outbuf,
 								  &contextattr,
 								  NULL);
+		pgstat_report_wait_end();
 
 		/* input buffer no longer used */
 		pfree(buf.data);
@@ -1392,11 +1400,13 @@ pg_SSPI_recvauth(Port *port)
 
 	CloseHandle(token);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_LOOKUP_ACCOUNT_SID);
 	if (!LookupAccountSid(NULL, tokenuser->User.Sid, accountname, &accountnamesize,
 						  domainname, &domainnamesize, &accountnameuse))
 		ereport(ERROR,
 				(errmsg_internal("could not look up account SID: error code %lu",
 								 GetLastError())));
+	pgstat_report_wait_end();
 
 	free(tokenuser);
 
@@ -1493,8 +1503,11 @@ pg_SSPI_make_upn(char *accountname,
 	 */
 
 	samname = psprintf("%s\\%s", domainname, accountname);
+
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						NULL, &upnamesize);
+	pgstat_report_wait_end();
 
 	if ((!res && GetLastError() != ERROR_INSUFFICIENT_BUFFER)
 		|| upnamesize == 0)
@@ -1509,8 +1522,10 @@ pg_SSPI_make_upn(char *accountname,
 	/* upnamesize includes the terminating NUL. */
 	upname = palloc(upnamesize);
 
+	pgstat_report_wait_start(WAIT_EVENT_SSPI_TRANSLATE_NAME);
 	res = TranslateName(samname, NameSamCompatible, NameUserPrincipal,
 						upname, &upnamesize);
+	pgstat_report_wait_end();
 
 	pfree(samname);
 	if (res)
@@ -2109,7 +2124,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_AUTHENTICATE);
 	retval = pam_authenticate(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2122,7 +2139,9 @@ CheckPAMAuth(Port *port, const char *user, const char *password)
 		return pam_no_password ? STATUS_EOF : STATUS_ERROR;
 	}
 
+	pgstat_report_wait_start(WAIT_EVENT_PAM_ACCT_MGMT);
 	retval = pam_acct_mgmt(pamh, 0);
+	pgstat_report_wait_end();
 
 	if (retval != PAM_SUCCESS)
 	{
@@ -2262,7 +2281,11 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 			}
 
 			/* Look up a list of LDAP server hosts and port numbers */
-			if (ldap_domain2hostlist(domain, &hostlist))
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_HOST_LOOKUP);
+			r = ldap_domain2hostlist(domain, &hostlist);
+			pgstat_report_wait_end();
+
+			if (r)
 			{
 				ereport(LOG,
 						(errmsg("LDAP authentication could not find DNS SRV records for \"%s\"",
@@ -2356,11 +2379,15 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 
 	if (port->hba->ldaptls)
 	{
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_START_TLS);
 #ifndef WIN32
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL);
 #else
-		if ((r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL)) != LDAP_SUCCESS)
+		r = ldap_start_tls_s(*ldap, NULL, NULL, NULL, NULL);
 #endif
+		pgstat_report_wait_end();
+
+		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
 					(errmsg("could not start LDAP TLS session: %s",
@@ -2520,9 +2547,12 @@ CheckLDAPAuth(Port *port)
 		 * Bind with a pre-defined username/password (if available) for
 		 * searching. If none is specified, this turns into an anonymous bind.
 		 */
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND_FOR_SEARCH);
 		r = ldap_simple_bind_s(ldap,
 							   port->hba->ldapbinddn ? port->hba->ldapbinddn : "",
 							   port->hba->ldapbindpasswd ? ldap_password_hook(port->hba->ldapbindpasswd) : "");
+		pgstat_report_wait_end();
+
 		if (r != LDAP_SUCCESS)
 		{
 			ereport(LOG,
@@ -2545,6 +2575,8 @@ CheckLDAPAuth(Port *port)
 			filter = psprintf("(uid=%s)", port->user_name);
 
 		search_message = NULL;
+
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_SEARCH);
 		r = ldap_search_s(ldap,
 						  port->hba->ldapbasedn,
 						  port->hba->ldapscope,
@@ -2552,6 +2584,7 @@ CheckLDAPAuth(Port *port)
 						  attributes,
 						  0,
 						  &search_message);
+		pgstat_report_wait_end();
 
 		if (r != LDAP_SUCCESS)
 		{
@@ -2620,7 +2653,9 @@ CheckLDAPAuth(Port *port)
 							port->user_name,
 							port->hba->ldapsuffix ? port->hba->ldapsuffix : "");
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_BIND);
 	r = ldap_simple_bind_s(ldap, fulluser, passwd);
+	pgstat_report_wait_end();
 
 	if (r != LDAP_SUCCESS)
 	{
@@ -3069,8 +3104,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		return STATUS_ERROR;
 	}
 
-	if (sendto(sock, radius_buffer, packetlength, 0,
-			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen) < 0)
+	pgstat_report_wait_start(WAIT_EVENT_RADIUS_SENDTO);
+	r = sendto(sock, radius_buffer, packetlength, 0,
+			   serveraddrs[0].ai_addr, serveraddrs[0].ai_addrlen);
+	pgstat_report_wait_end();
+
+	if (r < 0)
 	{
 		ereport(LOG,
 				(errmsg("could not send RADIUS packet: %m")));
@@ -3118,7 +3157,10 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		FD_ZERO(&fdset);
 		FD_SET(sock, &fdset);
 
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_WAIT);
 		r = select(sock + 1, &fdset, NULL, NULL, &timeout);
+		pgstat_report_wait_end();
+
 		if (r < 0)
 		{
 			if (errno == EINTR)
@@ -3151,8 +3193,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 
 		addrsize = sizeof(remoteaddr);
+
+		pgstat_report_wait_start(WAIT_EVENT_RADIUS_RECVFROM);
 		packetlength = recvfrom(sock, receive_buffer, RADIUS_BUFFER_SIZE, 0,
 								(struct sockaddr *) &remoteaddr, &addrsize);
+		pgstat_report_wait_end();
+
 		if (packetlength < 0)
 		{
 			ereport(LOG,
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index d9b8f34a355..6cc3e1e7c7a 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -34,6 +34,7 @@ static const char *pgstat_get_wait_client(WaitEventClient w);
 static const char *pgstat_get_wait_ipc(WaitEventIPC w);
 static const char *pgstat_get_wait_timeout(WaitEventTimeout w);
 static const char *pgstat_get_wait_io(WaitEventIO w);
+static const char *pgstat_get_wait_auth(WaitEventAuth w);
 
 
 static uint32 local_my_wait_event_info;
@@ -413,6 +414,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info)
 		case PG_WAIT_INJECTIONPOINT:
 			event_type = "InjectionPoint";
 			break;
+		case PG_WAIT_AUTH:
+			event_type = "Auth";
+			break;
 		default:
 			event_type = "???";
 			break;
@@ -495,6 +499,13 @@ pgstat_get_wait_event(uint32 wait_event_info)
 				event_name = pgstat_get_wait_io(w);
 				break;
 			}
+		case PG_WAIT_AUTH:
+			{
+				WaitEventAuth w = (WaitEventAuth) wait_event_info;
+
+				event_name = pgstat_get_wait_auth(w);
+				break;
+			}
 		default:
 			event_name = "unknown wait event";
 			break;
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index e199f071628..a2852225614 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -162,6 +162,33 @@ XACT_GROUP_UPDATE	"Waiting for the group leader to update transaction status at
 
 ABI_compatibility:
 
+#
+# Wait Events - Auth
+#
+# Use this category when a process is waiting for a third party to
+# authenticate/authorize the user.
+#
+
+Section: ClassName - WaitEventAuth
+
+GSSAPI_ACCEPT_SEC_CONTEXT	"Waiting for a response from a Kerberos server via GSSAPI."
+LDAP_BIND	"Waiting for an LDAP bind operation to authenticate the user."
+LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory."
+LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
+LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
+LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
+PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
+RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
+RADIUS_SENDTO	"Waiting for a <function>sendto</function> call during a RADIUS transaction."
+RADIUS_WAIT	"Waiting for a RADIUS server to respond."
+SSPI_ACCEPT_SECURITY_CONTEXT	"Waiting for a Windows security provider to accept the client's SSPI token."
+SSPI_ACQUIRE_CREDENTIALS_HANDLE	"Waiting for a Windows security provider to acquire server credentials for SSPI."
+SSPI_LOOKUP_ACCOUNT_SID	"Waiting for Windows to find the user's security identifier."
+SSPI_TRANSLATE_NAME	"Waiting for Windows to translate a Kerberos UPN."
+
+ABI_compatibility:
+
 #
 # Wait Events - Timeout
 #
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b8cb3e5a430..3d995a9e5be 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -25,6 +25,7 @@
 #define PG_WAIT_TIMEOUT				0x09000000U
 #define PG_WAIT_IO					0x0A000000U
 #define PG_WAIT_INJECTIONPOINT		0x0B000000U
+#define PG_WAIT_AUTH				0x0C000000U
 
 /* enums for wait events */
 #include "utils/wait_event_types.h"
diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out
index 83228cfca29..b94930658b0 100644
--- a/src/test/regress/expected/sysviews.out
+++ b/src/test/regress/expected/sysviews.out
@@ -181,6 +181,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
    type    | ok 
 -----------+----
  Activity  | t
+ Auth      | t
  BufferPin | t
  Client    | t
  Extension | t
@@ -189,7 +190,7 @@ select type, count(*) > 0 as ok FROM pg_wait_events
  LWLock    | t
  Lock      | t
  Timeout   | t
-(9 rows)
+(10 rows)
 
 -- Test that the pg_timezone_names and pg_timezone_abbrevs views are
 -- more-or-less working.  We can't test their contents in any great detail
-- 
2.34.1

From 15dd9dfcded95024f6e148d4a2a216ebb3e7ed1b Mon Sep 17 00:00:00 2001
From: Jacob Champion <jacob.champion@enterprisedb.com>
Date: Fri, 8 Nov 2024 14:19:26 -0800
Subject: [PATCH v11 2/2] squash! Report external auth calls as wait events

Add a wait event around all calls to ldap_unbind(). (For the record, I
do not want to implement this in this way.)
---
 src/backend/libpq/auth.c                       | 18 ++++++++++++++++++
 .../utils/activity/wait_event_names.txt        |  9 +++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index bd8a2a098b3..34cc145dc5e 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -2373,7 +2373,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 				(errmsg("could not set LDAP protocol version: %s",
 						ldap_err2string(r)),
 				 errdetail_for_ldap(*ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SET_OPTION);
 		ldap_unbind(*ldap);
+		pgstat_report_wait_end();
 		return STATUS_ERROR;
 	}
 
@@ -2393,7 +2395,9 @@ InitializeLDAPConnection(Port *port, LDAP **ldap)
 					(errmsg("could not start LDAP TLS session: %s",
 							ldap_err2string(r)),
 					 errdetail_for_ldap(*ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_START_TLS);
 			ldap_unbind(*ldap);
+			pgstat_report_wait_end();
 			return STATUS_ERROR;
 		}
 	}
@@ -2537,7 +2541,9 @@ CheckLDAPAuth(Port *port)
 			{
 				ereport(LOG,
 						(errmsg("invalid character in user name for LDAP authentication")));
+				pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_NAME_CHECK);
 				ldap_unbind(ldap);
+				pgstat_report_wait_end();
 				pfree(passwd);
 				return STATUS_ERROR;
 			}
@@ -2561,7 +2567,9 @@ CheckLDAPAuth(Port *port)
 							server_name,
 							ldap_err2string(r)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND_FOR_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			return STATUS_ERROR;
 		}
@@ -2594,7 +2602,9 @@ CheckLDAPAuth(Port *port)
 					 errdetail_for_ldap(ldap)));
 			if (search_message != NULL)
 				ldap_msgfree(search_message);
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_SEARCH);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			return STATUS_ERROR;
@@ -2616,7 +2626,9 @@ CheckLDAPAuth(Port *port)
 										  count,
 										  filter, server_name, count)));
 
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_COUNT_ENTRIES);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2635,7 +2647,9 @@ CheckLDAPAuth(Port *port)
 							filter, server_name,
 							ldap_err2string(error)),
 					 errdetail_for_ldap(ldap)));
+			pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_GET_DN);
 			ldap_unbind(ldap);
+			pgstat_report_wait_end();
 			pfree(passwd);
 			pfree(filter);
 			ldap_msgfree(search_message);
@@ -2663,7 +2677,9 @@ CheckLDAPAuth(Port *port)
 				(errmsg("LDAP login failed for user \"%s\" on server \"%s\": %s",
 						fulluser, server_name, ldap_err2string(r)),
 				 errdetail_for_ldap(ldap)));
+		pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_AFTER_BIND);
 		ldap_unbind(ldap);
+		pgstat_report_wait_end();
 		pfree(passwd);
 		pfree(fulluser);
 		return STATUS_ERROR;
@@ -2672,7 +2688,9 @@ CheckLDAPAuth(Port *port)
 	/* Save the original bind DN as the authenticated identity. */
 	set_authn_id(port, fulluser);
 
+	pgstat_report_wait_start(WAIT_EVENT_LDAP_UNBIND_SUCCESS);
 	ldap_unbind(ldap);
+	pgstat_report_wait_end();
 	pfree(passwd);
 	pfree(fulluser);
 
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index a2852225614..f082756c294 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -177,6 +177,15 @@ LDAP_BIND_FOR_SEARCH	"Waiting for an LDAP bind operation to search the directory
 LDAP_HOST_LOOKUP	"Waiting for DNS lookup of LDAP servers."
 LDAP_SEARCH	"Waiting for an LDAP search operation to complete."
 LDAP_START_TLS	"Waiting for an LDAP StartTLS exchange."
+LDAP_UNBIND_AFTER_BIND	"Waiting for an LDAP connection to be unbound after a simple bind failed."
+LDAP_UNBIND_AFTER_BIND_FOR_SEARCH	"Waiting for an LDAP connection to be unbound after a bind for search failed."
+LDAP_UNBIND_AFTER_COUNT_ENTRIES	"Waiting for an LDAP connection to be unbound after an entry count failed."
+LDAP_UNBIND_AFTER_GET_DN	"Waiting for an LDAP connection to be unbound after ldap_get_dn failed."
+LDAP_UNBIND_AFTER_NAME_CHECK	"Waiting for an LDAP connection to be unbound after a name check failed."
+LDAP_UNBIND_AFTER_SEARCH	"Waiting for an LDAP connection to be unbound after a bind+search failed."
+LDAP_UNBIND_AFTER_SET_OPTION	"Waiting for an LDAP connection to be unbound after ldap_set_option failed."
+LDAP_UNBIND_AFTER_START_TLS	"Waiting for an LDAP connection to be unbound after ldap_start_tls_s failed."
+LDAP_UNBIND_SUCCESS	"Waiting for a successful LDAP connection to be unbound."
 PAM_ACCT_MGMT	"Waiting for the PAM service to validate the user account."
 PAM_AUTHENTICATE	"Waiting for the PAM service to authenticate the user."
 RADIUS_RECVFROM	"Waiting for a <function>recvfrom</function> call during a RADIUS transaction."
-- 
2.34.1