UBSan pointer overflow in xlogreader.c

From d9ef9a66d8cec5ace83af8f55014759249b9ef77 Mon Sep 17 00:00:00 2001
From: Thomas Munro <thomas.munro@gmail.com>
Date: Wed, 22 Nov 2023 11:22:44 +1300
Subject: [PATCH] Fix pointer overflow in xlogreader.c.

While checking if a record could fit in the circular buffer, the coding
from commit 3f1ce973 used a formulation with pointer arithmetic that
could overflow.  This couldn't break on known 64 bit systems for various
technical reasons, which probably explains the lack of problem reports.
Likewise for known 32 bit kernels.  The systems at risk of problems
appear to be 32 bit processes running on 64 bit kernels.

Per complaint from ubsan.
---
 src/backend/access/transam/xlogreader.c | 46 ++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 9 deletions(-)

diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index e0baa86bd3..4dc6f06dd0 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -457,18 +457,37 @@ XLogReadRecordAlloc(XLogReaderState *state, size_t xl_tot_len, bool allow_oversi
 	if (state->decode_buffer_tail >= state->decode_buffer_head)
 	{
 		/* Empty, or tail is to the right of head. */
-		if (state->decode_buffer_tail + required_space <=
-			state->decode_buffer + state->decode_buffer_size)
+		if (required_space <=
+			state->decode_buffer_size -
+			(state->decode_buffer_tail - state->decode_buffer))
 		{
-			/* There is space between tail and end. */
+			/*-
+			 * There is space between tail and end.
+			 *
+			 * +-----+--------------------+-----+
+			 * |     |////////////////////|here!|
+			 * +-----+--------------------+-----+
+			 *       ^                    ^
+			 *       |                    |
+			 *       h                    t
+			 */
 			decoded = (DecodedXLogRecord *) state->decode_buffer_tail;
 			decoded->oversized = false;
 			return decoded;
 		}
-		else if (state->decode_buffer + required_space <
-				 state->decode_buffer_head)
+		else if (required_space <
+				 state->decode_buffer_head - state->decode_buffer)
 		{
-			/* There is space between start and head. */
+			/*-
+			 * There is space between start and head.
+			 *
+			 * +-----+--------------------+-----+
+			 * |here!|////////////////////|     |
+			 * +-----+--------------------+-----+
+			 *       ^                    ^
+			 *       |                    |
+			 *       h                    t
+			 */
 			decoded = (DecodedXLogRecord *) state->decode_buffer;
 			decoded->oversized = false;
 			return decoded;
@@ -477,10 +496,19 @@ XLogReadRecordAlloc(XLogReaderState *state, size_t xl_tot_len, bool allow_oversi
 	else
 	{
 		/* Tail is to the left of head. */
-		if (state->decode_buffer_tail + required_space <
-			state->decode_buffer_head)
+		if (required_space <
+			state->decode_buffer_head - state->decode_buffer_tail)
 		{
-			/* There is space between tail and head. */
+			/*-
+			 * There is space between tail and head.
+			 *
+			 * +-----+--------------------+-----+
+			 * |/////|here!               |/////|
+			 * +-----+--------------------+-----+
+			 *       ^                    ^
+			 *       |                    |
+			 *       t                    h
+			 */
 			decoded = (DecodedXLogRecord *) state->decode_buffer_tail;
 			decoded->oversized = false;
 			return decoded;
-- 
2.39.3 (Apple Git-145)