BUG #19555: TRAP: failed Assert (ent->cmin == change->data.tuplecid.cmin) on logical replication

Started by PG Bug reporting formabout 1 month ago3 messagesbugs
Jump to latest
#1PG Bug reporting form
noreply@postgresql.org

The following bug has been logged on the website:

Bug reference: 19555
Logged by: Alexander Kozhemyakin
Email address: a.kozhemyakin@postgrespro.ru
PostgreSQL version: 19beta2
Operating system: ubuntu 26.04
Description:

Hi,

We've encountered an assertion failure in ReorderBufferBuildTupleCidHash
when
processing DDL operations within subtransactions. The issue reproduces on
REL_19_STABLE (commit 64542957b44).

reproduce script:

psql -c "CREATE TABLE CHAR_TBL(f1 char(4));"
psql <<EOF
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot',
'test_decoding');
CREATE TABLE xact_test(data text);
INSERT INTO xact_test VALUES ('before-test');
EOF

psql -c "create type complex as (r float8, i float8);"
psql -c "CREATE TABLE base_tbl (a int PRIMARY KEY, b text DEFAULT
'Unspecified');"
psql -c "create table copydml_test (id serial, t text);"

psql <<EOF
CREATE TABLE stream_test(data text);
INSERT INTO stream_test SELECT repeat('a', 2000) || g.i FROM
generate_series(1, 1) g(i);
TRUNCATE table stream_test;
EOF

psql -c "checkpoint;"

psql <<EOF
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL,
'include-xids', '0', 'skip-empty-xacts', '1');
CREATE TABLE tr_sub_ddl(data int);
BEGIN;
SAVEPOINT a;
ALTER TABLE tr_sub_ddl ALTER COLUMN data TYPE text;
ROLLBACK TO SAVEPOINT a;
ALTER TABLE tr_sub_ddl ALTER COLUMN data TYPE bigint;
COMMIT;
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL,
'include-xids', '0', 'skip-empty-xacts', '1');
EOF

backtrace
(gdb) bt
#0 __pthread_kill_implementation (threadid=<optimized out>, signo=6,
no_tid=0) at ./nptl/pthread_kill.c:44
#1 __pthread_kill_internal (threadid=<optimized out>, signo=6) at
./nptl/pthread_kill.c:89
#2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at
./nptl/pthread_kill.c:100
#3 0x00007bfbec845b7e in __GI_raise (sig=sig@entry=6) at
../sysdeps/posix/raise.c:26
#4 0x00007bfbec8288ec in __GI_abort () at ./stdlib/abort.c:77
#5 0x000057ceaf77ce1c in ExceptionalCondition
(conditionName=conditionName@entry=0x57ceaf851c28 "ent->cmin ==
change->data.tuplecid.cmin",
fileName=fileName@entry=0x57ceaf8516c0
"/pgpro/postgres/src/backend/replication/logical/reorderbuffer.c",
lineNumber=lineNumber@entry=1890) at
/pgpro/postgres/src/backend/utils/error/assert.c:65
#6 0x000057ceaf56e58b in ReorderBufferBuildTupleCidHash (rb=0x57ceb31da6f0,
txn=0x57ceb31ee6d0) at
/pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:1890
#7 ReorderBufferProcessTXN (rb=0x57ceb31da6f0, txn=0x57ceb31ee6d0,
commit_lsn=25223904, snapshot_now=<optimized out>,
command_id=command_id@entry=0, streaming=streaming@entry=false)
at /pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:2220
#8 0x000057ceaf56f61c in ReorderBufferReplay (txn=<optimized out>,
rb=<optimized out>, commit_lsn=<optimized out>, end_lsn=<optimized out>,
commit_time=<optimized out>, origin_id=<optimized out>, origin_lsn=0,
xid=<optimized out>) at
/pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:2870
#9 0x000057ceaf570375 in ReorderBufferCommit (rb=<optimized out>,
xid=<optimized out>, commit_lsn=<optimized out>, end_lsn=<optimized out>,
commit_time=<optimized out>, origin_id=<optimized out>,
origin_lsn=<optimized out>)
at /pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:2894
#10 0x000057ceaf55db32 in DecodeCommit (ctx=0x57ceb31d66d0,
buf=0x7ffe6ad14500, parsed=0x7ffe6ad14370, xid=702, two_phase=false) at
/pgpro/postgres/src/backend/replication/logical/decode.c:755
#11 xact_decode (ctx=0x57ceb31d66d0, buf=0x7ffe6ad14500) at
/pgpro/postgres/src/backend/replication/logical/decode.c:254
#12 0x000057ceaf55d473 in LogicalDecodingProcessRecord
(ctx=ctx@entry=0x57ceb31d66d0, record=0x57ceb31d6af8) at
/pgpro/postgres/src/backend/replication/logical/decode.c:117
#13 0x000057ceaf564d37 in pg_logical_slot_get_changes_guts
(fcinfo=0x57ceb31c6660, confirm=confirm@entry=true,
binary=binary@entry=false) at
/pgpro/postgres/src/backend/replication/logical/logicalfuncs.c:267
#14 0x000057ceaf564ec4 in pg_logical_slot_get_changes (fcinfo=<optimized
out>) at /pgpro/postgres/src/backend/replication/logical/logicalfuncs.c:336
#15 0x000057ceaf3f3bcd in ExecMakeTableFunctionResult
(setexpr=0x57ceb31bebd8, econtext=0x57ceb31bea28, argContext=<optimized
out>, expectedDesc=0x57ceb31d8618, randomAccess=false)
at /pgpro/postgres/src/backend/executor/execSRF.c:235
#16 0x000057ceaf408c7d in FunctionNext (node=node@entry=0x57ceb31be818) at
/pgpro/postgres/src/backend/executor/nodeFunctionscan.c:95
#17 0x000057ceaf3f44ac in ExecScanFetch (node=<optimized out>,
epqstate=<optimized out>, accessMtd=<optimized out>, recheckMtd=<optimized
out>) at /pgpro/postgres/src/include/executor/execScan.h:135
#18 ExecScanExtended (node=<optimized out>, accessMtd=0x57ceaf408950
<FunctionNext>, recheckMtd=0x57ceaf408940 <FunctionRecheck>, epqstate=0x0,
qual=0x0, projInfo=0x57ceb31d8c58)
at /pgpro/postgres/src/include/executor/execScan.h:196
#19 ExecScan (node=0x57ceb31be818, accessMtd=0x57ceaf408950 <FunctionNext>,
recheckMtd=0x57ceaf408940 <FunctionRecheck>) at
/pgpro/postgres/src/backend/executor/execScan.c:59
#20 0x000057ceaf3e852b in ExecProcNode (node=0x57ceb31be818) at
/pgpro/postgres/src/include/executor/executor.h:327
#21 ExecutePlan (queryDesc=0x57ceb319b2e0, operation=CMD_SELECT,
sendTuples=true, numberTuples=0, direction=<optimized out>,
dest=0x57ceb31c0cc8) at /pgpro/postgres/src/backend/executor/execMain.c:1745
#22 standard_ExecutorRun (queryDesc=0x57ceb319b2e0, direction=<optimized
out>, count=0) at /pgpro/postgres/src/backend/executor/execMain.c:377
#23 0x000057ceaf60bc98 in PortalRunSelect
(portal=portal@entry=0x57ceb314ed40, forward=forward@entry=true, count=0,
count@entry=9223372036854775807, dest=dest@entry=0x57ceb31c0cc8)
at /pgpro/postgres/src/backend/tcop/pquery.c:917
#24 0x000057ceaf60d35e in PortalRun (portal=portal@entry=0x57ceb314ed40,
count=count@entry=9223372036854775807, isTopLevel=isTopLevel@entry=true,
dest=dest@entry=0x57ceb31c0cc8, altdest=altdest@entry=0x57ceb31c0cc8,
qc=qc@entry=0x7ffe6ad14b10) at
/pgpro/postgres/src/backend/tcop/pquery.c:761
#25 0x000057ceaf608e5a in exec_simple_query (query_string=0x57ceb30a0220
"SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL,
'include-xids', '0', 'skip-empty-xacts', '1');")
at /pgpro/postgres/src/backend/tcop/postgres.c:1290
#26 0x000057ceaf60aa5d in PostgresMain (dbname=<optimized out>,
username=<optimized out>) at
/pgpro/postgres/src/backend/tcop/postgres.c:4856
#27 0x000057ceaf604a03 in BackendMain (startup_data=<optimized out>,
startup_data_len=<optimized out>) at
/pgpro/postgres/src/backend/tcop/backend_startup.c:124
#28 0x000057ceaf53f461 in postmaster_child_launch (child_type=<optimized
out>, child_slot=1, startup_data=startup_data@entry=0x7ffe6ad14fe0,
startup_data_len=startup_data_len@entry=24,
client_sock=client_sock@entry=0x7ffe6ad15000) at
/pgpro/postgres/src/backend/postmaster/launch_backend.c:268
#29 0x000057ceaf543184 in BackendStartup (client_sock=0x7ffe6ad15000) at
/pgpro/postgres/src/backend/postmaster/postmaster.c:3627
#30 ServerLoop () at
/pgpro/postgres/src/backend/postmaster/postmaster.c:1728
#31 0x000057ceaf544e15 in PostmasterMain (argc=argc@entry=3,
argv=argv@entry=0x57ceb3047570) at
/pgpro/postgres/src/backend/postmaster/postmaster.c:1415
#32 0x000057ceaf1ceb2e in main (argc=3, argv=0x57ceb3047570) at
/pgpro/postgres/src/backend/main/main.c:231

#2Andrey Rachitskiy
pl0h0yp1@gmail.com
In reply to: PG Bug reporting form (#1)
Re: BUG #19555: TRAP: failed Assert (ent->cmin == change->data.tuplecid.cmin) on logical replication

Hi, Alexander!

We've hit the adjacent cmin assertion in ReorderBufferBuildTupleCidHash

TRAP: failed Assert("ent->cmin == change->data.tuplecid.cmin"),
File: "reorderbuffer.c", Line: 1890

This is not a newly discovered bug.
It is the unfinished follow-up to the 2019 cmax fix:
Arseny Sher already diagnosed the cmin case, posted a fix
(pick_latest_cmin_in_reorderbuffer.patch), and later reminded that it was
still open — but that half was never committed.
What we see on current REL_19_STABLE is the same open issue.

History
-----------
In early 2019 Arseny reported that the cmax assert in
ReorderBufferBuildTupleCidHash was too strict when DDL ran in an aborted
subtransaction and the same catalog tuple was touched again later in the
same top-level xact:

/messages/by-id/874l9p8hyw.fsf@ars-thinkpad

Alvaro committed a relaxation of the cmax assert (keep it valid and
monotonic) as:

https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=8c67d29fd51c0381d8bce41d35d7726725924616;hp=d357a16997a2e9dce0f56299c739b2b2584c4026

Commit announcement:
/messages/by-id/E1gtg5N-0005RZ-9n@gemulon.postgresql.org

A few days later Arseny pointed out that the neighbouring cmin assert is
also wrong: after an aborted subxact insert is pruned, the same
(relfilenode, tid) can refer to a completely different physical tuple with
a different cmin. He proposed always keeping the latest mapping
(pick_latest_cmin_in_reorderbuffer.patch):

/messages/by-id/87r2c8xocq.fsf@ars-thinkpad

and reminded about it later:

/messages/by-id/87lfr8wa5p.fsf@ars-thinkpad

That cmin half never landed. HEAD still has Assert(ent->cmin ==
change->data.tuplecid.cmin) and on duplicate keys only updates cmax. So
this remains an old, unclosed tail of that thread: the cmax side was fixed
in 2019, the cmin / "pick latest" side was proposed then and left
uncommitted.

More recently Mark Dilger raised the same issue again and proposed a fix
(fix-tuplecid-cmin-upstream.patch):

/messages/by-id/CAHgHdKu5e3XY5e90Tuaxq_R4WrKxSV734Q+Lwo5y39Omp2A-Gg@mail.gmail.com

---
Regards,
Rachitskiy Andrey

пт, 17 июл. 2026 г. в 21:30, PG Bug reporting form <noreply@postgresql.org>:

Show quoted text

The following bug has been logged on the website:

Bug reference: 19555
Logged by: Alexander Kozhemyakin
Email address: a.kozhemyakin@postgrespro.ru
PostgreSQL version: 19beta2
Operating system: ubuntu 26.04
Description:

Hi,

We've encountered an assertion failure in ReorderBufferBuildTupleCidHash
when
processing DDL operations within subtransactions. The issue reproduces on
REL_19_STABLE (commit 64542957b44).

reproduce script:

psql -c "CREATE TABLE CHAR_TBL(f1 char(4));"
psql <<EOF
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot',
'test_decoding');
CREATE TABLE xact_test(data text);
INSERT INTO xact_test VALUES ('before-test');
EOF

psql -c "create type complex as (r float8, i float8);"
psql -c "CREATE TABLE base_tbl (a int PRIMARY KEY, b text DEFAULT
'Unspecified');"
psql -c "create table copydml_test (id serial, t text);"

psql <<EOF
CREATE TABLE stream_test(data text);
INSERT INTO stream_test SELECT repeat('a', 2000) || g.i FROM
generate_series(1, 1) g(i);
TRUNCATE table stream_test;
EOF

psql -c "checkpoint;"

psql <<EOF
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL,
'include-xids', '0', 'skip-empty-xacts', '1');
CREATE TABLE tr_sub_ddl(data int);
BEGIN;
SAVEPOINT a;
ALTER TABLE tr_sub_ddl ALTER COLUMN data TYPE text;
ROLLBACK TO SAVEPOINT a;
ALTER TABLE tr_sub_ddl ALTER COLUMN data TYPE bigint;
COMMIT;
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL,
'include-xids', '0', 'skip-empty-xacts', '1');
EOF

backtrace
(gdb) bt
#0 __pthread_kill_implementation (threadid=<optimized out>, signo=6,
no_tid=0) at ./nptl/pthread_kill.c:44
#1 __pthread_kill_internal (threadid=<optimized out>, signo=6) at
./nptl/pthread_kill.c:89
#2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at
./nptl/pthread_kill.c:100
#3 0x00007bfbec845b7e in __GI_raise (sig=sig@entry=6) at
../sysdeps/posix/raise.c:26
#4 0x00007bfbec8288ec in __GI_abort () at ./stdlib/abort.c:77
#5 0x000057ceaf77ce1c in ExceptionalCondition
(conditionName=conditionName@entry=0x57ceaf851c28 "ent->cmin ==
change->data.tuplecid.cmin",
fileName=fileName@entry=0x57ceaf8516c0
"/pgpro/postgres/src/backend/replication/logical/reorderbuffer.c",
lineNumber=lineNumber@entry=1890) at
/pgpro/postgres/src/backend/utils/error/assert.c:65
#6 0x000057ceaf56e58b in ReorderBufferBuildTupleCidHash
(rb=0x57ceb31da6f0,
txn=0x57ceb31ee6d0) at
/pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:1890
#7 ReorderBufferProcessTXN (rb=0x57ceb31da6f0, txn=0x57ceb31ee6d0,
commit_lsn=25223904, snapshot_now=<optimized out>,
command_id=command_id@entry=0, streaming=streaming@entry=false)
at /pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:2220
#8 0x000057ceaf56f61c in ReorderBufferReplay (txn=<optimized out>,
rb=<optimized out>, commit_lsn=<optimized out>, end_lsn=<optimized out>,
commit_time=<optimized out>, origin_id=<optimized out>, origin_lsn=0,
xid=<optimized out>) at
/pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:2870
#9 0x000057ceaf570375 in ReorderBufferCommit (rb=<optimized out>,
xid=<optimized out>, commit_lsn=<optimized out>, end_lsn=<optimized out>,
commit_time=<optimized out>, origin_id=<optimized out>,
origin_lsn=<optimized out>)
at /pgpro/postgres/src/backend/replication/logical/reorderbuffer.c:2894
#10 0x000057ceaf55db32 in DecodeCommit (ctx=0x57ceb31d66d0,
buf=0x7ffe6ad14500, parsed=0x7ffe6ad14370, xid=702, two_phase=false) at
/pgpro/postgres/src/backend/replication/logical/decode.c:755
#11 xact_decode (ctx=0x57ceb31d66d0, buf=0x7ffe6ad14500) at
/pgpro/postgres/src/backend/replication/logical/decode.c:254
#12 0x000057ceaf55d473 in LogicalDecodingProcessRecord
(ctx=ctx@entry=0x57ceb31d66d0, record=0x57ceb31d6af8) at
/pgpro/postgres/src/backend/replication/logical/decode.c:117
#13 0x000057ceaf564d37 in pg_logical_slot_get_changes_guts
(fcinfo=0x57ceb31c6660, confirm=confirm@entry=true,
binary=binary@entry=false) at
/pgpro/postgres/src/backend/replication/logical/logicalfuncs.c:267
#14 0x000057ceaf564ec4 in pg_logical_slot_get_changes (fcinfo=<optimized
out>) at /pgpro/postgres/src/backend/replication/logical/logicalfuncs.c:336
#15 0x000057ceaf3f3bcd in ExecMakeTableFunctionResult
(setexpr=0x57ceb31bebd8, econtext=0x57ceb31bea28, argContext=<optimized
out>, expectedDesc=0x57ceb31d8618, randomAccess=false)
at /pgpro/postgres/src/backend/executor/execSRF.c:235
#16 0x000057ceaf408c7d in FunctionNext (node=node@entry=0x57ceb31be818) at
/pgpro/postgres/src/backend/executor/nodeFunctionscan.c:95
#17 0x000057ceaf3f44ac in ExecScanFetch (node=<optimized out>,
epqstate=<optimized out>, accessMtd=<optimized out>, recheckMtd=<optimized
out>) at /pgpro/postgres/src/include/executor/execScan.h:135
#18 ExecScanExtended (node=<optimized out>, accessMtd=0x57ceaf408950
<FunctionNext>, recheckMtd=0x57ceaf408940 <FunctionRecheck>, epqstate=0x0,
qual=0x0, projInfo=0x57ceb31d8c58)
at /pgpro/postgres/src/include/executor/execScan.h:196
#19 ExecScan (node=0x57ceb31be818, accessMtd=0x57ceaf408950 <FunctionNext>,
recheckMtd=0x57ceaf408940 <FunctionRecheck>) at
/pgpro/postgres/src/backend/executor/execScan.c:59
#20 0x000057ceaf3e852b in ExecProcNode (node=0x57ceb31be818) at
/pgpro/postgres/src/include/executor/executor.h:327
#21 ExecutePlan (queryDesc=0x57ceb319b2e0, operation=CMD_SELECT,
sendTuples=true, numberTuples=0, direction=<optimized out>,
dest=0x57ceb31c0cc8) at
/pgpro/postgres/src/backend/executor/execMain.c:1745
#22 standard_ExecutorRun (queryDesc=0x57ceb319b2e0, direction=<optimized
out>, count=0) at /pgpro/postgres/src/backend/executor/execMain.c:377
#23 0x000057ceaf60bc98 in PortalRunSelect
(portal=portal@entry=0x57ceb314ed40, forward=forward@entry=true, count=0,
count@entry=9223372036854775807, dest=dest@entry=0x57ceb31c0cc8)
at /pgpro/postgres/src/backend/tcop/pquery.c:917
#24 0x000057ceaf60d35e in PortalRun (portal=portal@entry=0x57ceb314ed40,
count=count@entry=9223372036854775807, isTopLevel=isTopLevel@entry=true,
dest=dest@entry=0x57ceb31c0cc8, altdest=altdest@entry=0x57ceb31c0cc8,
qc=qc@entry=0x7ffe6ad14b10) at
/pgpro/postgres/src/backend/tcop/pquery.c:761
#25 0x000057ceaf608e5a in exec_simple_query (query_string=0x57ceb30a0220
"SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL,
NULL,
'include-xids', '0', 'skip-empty-xacts', '1');")
at /pgpro/postgres/src/backend/tcop/postgres.c:1290
#26 0x000057ceaf60aa5d in PostgresMain (dbname=<optimized out>,
username=<optimized out>) at
/pgpro/postgres/src/backend/tcop/postgres.c:4856
#27 0x000057ceaf604a03 in BackendMain (startup_data=<optimized out>,
startup_data_len=<optimized out>) at
/pgpro/postgres/src/backend/tcop/backend_startup.c:124
#28 0x000057ceaf53f461 in postmaster_child_launch (child_type=<optimized
out>, child_slot=1, startup_data=startup_data@entry=0x7ffe6ad14fe0,
startup_data_len=startup_data_len@entry=24,
client_sock=client_sock@entry=0x7ffe6ad15000) at
/pgpro/postgres/src/backend/postmaster/launch_backend.c:268
#29 0x000057ceaf543184 in BackendStartup (client_sock=0x7ffe6ad15000) at
/pgpro/postgres/src/backend/postmaster/postmaster.c:3627
#30 ServerLoop () at
/pgpro/postgres/src/backend/postmaster/postmaster.c:1728
#31 0x000057ceaf544e15 in PostmasterMain (argc=argc@entry=3,
argv=argv@entry=0x57ceb3047570) at
/pgpro/postgres/src/backend/postmaster/postmaster.c:1415
#32 0x000057ceaf1ceb2e in main (argc=3, argv=0x57ceb3047570) at
/pgpro/postgres/src/backend/main/main.c:231

#3Bingshuai Li
lucian1412@outlook.com
In reply to: PG Bug reporting form (#1)
Re: BUG #19555: TRAP: failed Assert (ent->cmin == change->data.tuplecid.cmin) on logical replication

As Andrey noted, this report is the cmin half of the issue whose
cmax half was fixed in 8c67d29fd51. Confirmed on current master
(36f7330b8b2) with a slightly different table shape, and there is
an existing patch on pgsql-hackers [1]/messages/by-id/CAHgHdKu5e3XY5e90Tuaxq_R4WrKxSV734Q+Lwo5y39Omp2A-Gg@mail.gmail.com, which fixes the crash in
my testing. I have posted reproduction results, a WAL-level
confirmation of the diagnosis, and review comments in that
thread [2]/messages/by-id/ME0P300MB05682FD742273EC64362A8A1C6D42@ME0P300MB0568.AUSP300.PROD.OUTLOOK.COM.

[1]: /messages/by-id/CAHgHdKu5e3XY5e90Tuaxq_R4WrKxSV734Q+Lwo5y39Omp2A-Gg@mail.gmail.com
[2]: /messages/by-id/ME0P300MB05682FD742273EC64362A8A1C6D42@ME0P300MB0568.AUSP300.PROD.OUTLOOK.COM

Thanks,
Bingshuai Li