9.4 HEAD: select() failed in postmaster

diff -pcdr old/doc/src/sgml/runtime.sgml new/doc/src/sgml/runtime.sgml
*** old/doc/src/sgml/runtime.sgml	2013-02-04 09:17:04.000000000 +0900
--- new/doc/src/sgml/runtime.sgml	2013-02-07 23:13:28.626077200 +0900
*************** echo -1000 > /proc/self/oom_score_adj
*** 1363,1372 ****
        <para>
        This is the <firstterm>Immediate Shutdown</firstterm> mode.
        The master <command>postgres</command> process will send a
!       <systemitem>SIGQUIT</systemitem> to all child processes and exit
!       immediately, without properly shutting itself down. The child processes
!       likewise exit immediately upon receiving
!       <systemitem>SIGQUIT</systemitem>. This will lead to recovery (by
        replaying the WAL log) upon next start-up. This is recommended
        only in emergencies.
        </para>
--- 1363,1376 ----
        <para>
        This is the <firstterm>Immediate Shutdown</firstterm> mode.
        The master <command>postgres</command> process will send a
!       <systemitem>SIGQUIT</systemitem> to all child processes, wait for
!       them to terminate, and exit. The child processes
!       exit immediately upon receiving
!       <systemitem>SIGQUIT</systemitem>. If any of the child processes
!       does not terminate within 10 seconds for some unexpected reason,
!       the master postgres process will send a <systemitem>SIGKILL</systemitem>
!       to all remaining ones, wait for their termination
!       again, and exit. This will lead to recovery (by
        replaying the WAL log) upon next start-up. This is recommended
        only in emergencies.
        </para>
diff -pcdr old/src/backend/postmaster/postmaster.c new/src/backend/postmaster/postmaster.c
*** old/src/backend/postmaster/postmaster.c	2013-02-04 09:17:04.000000000 +0900
--- new/src/backend/postmaster/postmaster.c	2013-02-07 23:13:28.987077200 +0900
*************** static pid_t StartupPID = 0,
*** 273,284 ****
--- 273,288 ----
  #define			NoShutdown		0
  #define			SmartShutdown	1
  #define			FastShutdown	2
+ #define			ImmediateShutdown	3
  
  static int	Shutdown = NoShutdown;
  
  static bool FatalError = false; /* T if recovering from backend crash */
  static bool RecoveryError = false;		/* T if WAL recovery failed */
  
+ /* Start time of abort processing at immediate shutdown or child crash */
+ static time_t AbortStartTime;
+ 
  /*
   * We use a simple state machine to control startup, shutdown, and
   * crash recovery (which is rather like shutdown followed by startup).
*************** static void RandomSalt(char *md5Salt);
*** 419,424 ****
--- 423,429 ----
  static void signal_child(pid_t pid, int signal);
  static bool SignalSomeChildren(int signal, int targets);
  static bool SignalUnconnectedWorkers(int signal);
+ static void SignalAllChildren(int signal);
  
  #define SignalChildren(sig)			   SignalSomeChildren(sig, BACKEND_TYPE_ALL)
  
*************** ServerLoop(void)
*** 1658,1663 ****
--- 1663,1688 ----
  			TouchSocketLockFiles();
  			last_touch_time = now;
  		}
+ 
+ 		/*
+ 		 * When postmaster got an immediate shutdown request
+ 		 * or some child terminated abnormally (FatalError case),
+ 		 * postmaster sends SIGQUIT to all children except
+ 		 * syslogger and dead_end ones, then wait for them to terminate.
+ 		 * If some children didn't terminate within a certain amount of time,
+ 		 * postmaster sends SIGKILL to them and wait again.
+ 		 * This resolves, for example, the hang situation where
+ 		 * a backend gets stuck in the call chain:
+ 		 * free() acquires some lock -> <received SIGQUIT> ->
+ 		 * quickdie() -> ereport() -> gettext() -> malloc() -> <lock acquisition>
+ 		 */
+ 		if (AbortStartTime > 0 &&  /* SIGKILL only once */
+ 			(Shutdown == ImmediateShutdown || (FatalError && !SendStop)) &&
+ 			now - AbortStartTime >= 10)
+ 		{
+ 			SignalAllChildren(SIGKILL);
+ 			AbortStartTime = 0;
+ 		}
  	}
  }
  
*************** pmdie(SIGNAL_ARGS)
*** 2452,2481 ****
  			/*
  			 * Immediate Shutdown:
  			 *
! 			 * abort all children with SIGQUIT and exit without attempt to
! 			 * properly shut down data base system.
  			 */
  			ereport(LOG,
  					(errmsg("received immediate shutdown request")));
! 			SignalChildren(SIGQUIT);
! 			if (StartupPID != 0)
! 				signal_child(StartupPID, SIGQUIT);
! 			if (BgWriterPID != 0)
! 				signal_child(BgWriterPID, SIGQUIT);
! 			if (CheckpointerPID != 0)
! 				signal_child(CheckpointerPID, SIGQUIT);
! 			if (WalWriterPID != 0)
! 				signal_child(WalWriterPID, SIGQUIT);
! 			if (WalReceiverPID != 0)
! 				signal_child(WalReceiverPID, SIGQUIT);
! 			if (AutoVacPID != 0)
! 				signal_child(AutoVacPID, SIGQUIT);
! 			if (PgArchPID != 0)
! 				signal_child(PgArchPID, SIGQUIT);
! 			if (PgStatPID != 0)
! 				signal_child(PgStatPID, SIGQUIT);
! 			SignalUnconnectedWorkers(SIGQUIT);
! 			ExitPostmaster(0);
  			break;
  	}
  
--- 2477,2501 ----
  			/*
  			 * Immediate Shutdown:
  			 *
! 			 * abort all children with SIGQUIT, wait for all children to
! 			 * terminate, and exit without attempt to properly
! 			 * shut down data base system.
  			 */
+ 			if (Shutdown >= ImmediateShutdown)
+ 				break;
+ 			Shutdown = ImmediateShutdown;
  			ereport(LOG,
  					(errmsg("received immediate shutdown request")));
! 
! 			SignalAllChildren(SIGQUIT);
! 			pmState = PM_WAIT_BACKENDS;
! 			AbortStartTime = time(NULL);
! 
! 			/*
! 			 * Now wait for backends to exit.  If there are none,
! 			 * PostmasterStateMachine will take the next step.
! 			 */
! 			PostmasterStateMachine();
  			break;
  	}
  
*************** HandleChildCrash(int pid, int exitstatus
*** 2950,2955 ****
--- 2970,2979 ----
  	slist_iter	siter;
  	Backend    *bp;
  
+ 	/* Do nothing if the child terminated due to immediate shutdown */
+ 	if (Shutdown == ImmediateShutdown)
+ 		return;
+ 
  	/*
  	 * Make log entry unless there was a previous crash (if so, nonzero exit
  	 * status is to be expected in SIGQUIT response; don't clutter log)
*************** HandleChildCrash(int pid, int exitstatus
*** 3177,3182 ****
--- 3201,3207 ----
  		pmState == PM_WAIT_READONLY ||
  		pmState == PM_SHUTDOWN)
  		pmState = PM_WAIT_BACKENDS;
+ 	AbortStartTime = time(NULL);
  }
  
  /*
*************** PostmasterStateMachine(void)
*** 3313,3319 ****
  			WalWriterPID == 0 &&
  			AutoVacPID == 0)
  		{
! 			if (FatalError)
  			{
  				/*
  				 * Start waiting for dead_end children to die.	This state
--- 3338,3344 ----
  			WalWriterPID == 0 &&
  			AutoVacPID == 0)
  		{
! 			if (Shutdown == ImmediateShutdown || FatalError)
  			{
  				/*
  				 * Start waiting for dead_end children to die.	This state
*************** PostmasterStateMachine(void)
*** 3323,3329 ****
  
  				/*
  				 * We already SIGQUIT'd the archiver and stats processes, if
! 				 * any, when we entered FatalError state.
  				 */
  			}
  			else
--- 3348,3355 ----
  
  				/*
  				 * We already SIGQUIT'd the archiver and stats processes, if
! 				 * any, when we started immediate shutdown or entered
! 				 * FatalError state.
  				 */
  			}
  			else
*************** signal_child(pid_t pid, int signal)
*** 3508,3513 ****
--- 3534,3540 ----
  		case SIGTERM:
  		case SIGQUIT:
  		case SIGSTOP:
+ 		case SIGKILL:
  			if (kill(-pid, signal) < 0)
  				elog(DEBUG3, "kill(%ld,%d) failed: %m", (long) (-pid), signal);
  			break;
*************** SignalSomeChildren(int signal, int targe
*** 3595,3600 ****
--- 3622,3654 ----
  }
  
  /*
+  * Send a signal to all children (but NOT syslogger;
+  * dead_end children are never signaled, either).
+  */
+ static void
+ SignalAllChildren(int signal)
+ {
+ 	SignalChildren(signal);
+ 	if (StartupPID != 0)
+ 		signal_child(StartupPID, signal);
+ 	if (BgWriterPID != 0)
+ 		signal_child(BgWriterPID, signal);
+ 	if (CheckpointerPID != 0)
+ 		signal_child(CheckpointerPID, signal);
+ 	if (WalWriterPID != 0)
+ 		signal_child(WalWriterPID, signal);
+ 	if (WalReceiverPID != 0)
+ 		signal_child(WalReceiverPID, signal);
+ 	if (AutoVacPID != 0)
+ 		signal_child(AutoVacPID, signal);
+ 	if (PgArchPID != 0)
+ 		signal_child(PgArchPID, signal);
+ 	if (PgStatPID != 0)
+ 		signal_child(PgStatPID, signal);
+ 	SignalUnconnectedWorkers(signal);
+ }
+ 
+ /*
   * BackendStartup -- start backend process
   *
   * returns: STATUS_ERROR if the fork failed, STATUS_OK otherwise.
diff -pcdr old/src/port/kill.c new/src/port/kill.c
*** old/src/port/kill.c	2013-02-04 09:17:04.000000000 +0900
--- new/src/port/kill.c	2013-02-07 23:13:29.066077200 +0900
*************** pgkill(int pid, int sig)
*** 38,43 ****
--- 38,61 ----
  		errno = EINVAL;
  		return -1;
  	}
+ 	if (sig == SIGKILL)
+ 	{
+ 		HANDLE prochandle;
+ 
+ 		if ((prochandle = OpenProcess(PROCESS_TERMINATE, FALSE, (DWORD) pid)) == NULL)
+ 		{
+ 			errno = ESRCH;
+ 			return -1;
+ 		}
+ 		if (!TerminateProcess(prochandle, 255))
+ 		{
+ 			_dosmaperr(GetLastError());
+ 			CloseHandle(prochandle);
+ 			return -1;
+ 		}
+ 		CloseHandle(prochandle);
+ 		return 0;
+ 	}
  	snprintf(pipename, sizeof(pipename), "\\\\.\\pipe\\pgsignal_%u", pid);
  
  	if (CallNamedPipe(pipename, &sigData, 1, &sigRet, 1, &bytes, 1000))

diff -rpcd a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
*** a/src/backend/postmaster/postmaster.c	2013-10-04 13:17:03.000000000 +0900
--- b/src/backend/postmaster/postmaster.c	2013-10-04 23:21:39.541000000 +0900
*************** DetermineSleepTime(struct timeval * time
*** 1423,1429 ****
  		if (AbortStartTime > 0)
  		{
  			/* remaining time, but at least 1 second */
! 			timeout->tv_sec = Min(SIGKILL_CHILDREN_AFTER_SECS -
  								  (time(NULL) - AbortStartTime), 1);
  			timeout->tv_usec = 0;
  		}
--- 1423,1429 ----
  		if (AbortStartTime > 0)
  		{
  			/* remaining time, but at least 1 second */
! 			timeout->tv_sec = Max(SIGKILL_CHILDREN_AFTER_SECS -
  								  (time(NULL) - AbortStartTime), 1);
  			timeout->tv_usec = 0;
  		}
*************** ServerLoop(void)
*** 1676,1685 ****
--- 1676,1687 ----
  		 * Note we also do this during recovery from a process crash.
  		 */
  		if ((Shutdown >= ImmediateShutdown || (FatalError && !SendStop)) &&
+ 			AbortStartTime > 0 &&  /* SIGKILL only once */
  			now - AbortStartTime >= SIGKILL_CHILDREN_AFTER_SECS)
  		{
  			/* We were gentle with them before. Not anymore */
  			TerminateChildren(SIGKILL);
+ 			AbortStartTime = 0;
  
  			/*
  			 * Additionally, unless we're recovering from a process crash, it's
*************** reaper(SIGNAL_ARGS)
*** 2584,2590 ****
  			 * Startup succeeded, commence normal operations
  			 */
  			FatalError = false;
- 			AbortStartTime = 0;
  			ReachedNormalRunning = true;
  			pmState = PM_RUN;
  
--- 2586,2591 ----
*************** PostmasterStateMachine(void)
*** 3544,3549 ****
--- 3545,3551 ----
  		StartupPID = StartupDataBase();
  		Assert(StartupPID != 0);
  		pmState = PM_STARTUP;
+ 		AbortStartTime = 0;
  	}
  }
  
*************** sigusr1_handler(SIGNAL_ARGS)
*** 4737,4743 ****
  	{
  		/* WAL redo has started. We're out of reinitialization. */
  		FatalError = false;
- 		AbortStartTime = 0;
  
  		/*
  		 * Crank up the background tasks.  It doesn't matter if this fails,
--- 4739,4744 ----