BUG #16219: EvalPlanQualFetchRowMark segfaults on Updates

The following bug has been logged on the website:

Bug reference: 16219
Logged by: Julian Schauder
Email address: julian.schauder@gmx.de
PostgreSQL version: 12.1
Operating system: Ubuntu 18.04
Description:

Howdy Hackers,

for a week now (upgrade pg11=>12) we're experiencing a bug that causes
segfaults on a daily basis.
I've yet to reproduce this exact behaviour as the querys do not fail all the
time but rather now-and-then
when run concurrently. ( ~4 Crashes so far )

Outputs(identifiers) are anonymised.

2020-01-20 01:39:24.246 CET [2112] LOG: server process (PID 31599) was

terminated by signal 11: Segmentation fault

2020-01-20 01:39:24.246 CET [2112] DETAIL: Failed process was running:

UPDATE partitionedtable ia SET A=A, B=now() FROM temptable tt WHERE
ia.id=tt.ia_id AND A::text IS DISTINCT FROM tt.A::text;

2020-01-20 01:39:24.246 CET [2112] LOG: terminating any other active

server processes

#0 EvalPlanQualFetchRowMark (epqstate=0x5576bfbccfc0, rti=2,

slot=slot@entry=0x5576c888b6c0) at
./build/../src/backend/executor/execMain.c:2590

#1 0x00005576bdf449ce in ExecScanFetch (recheckMtd=0x5576bdf65740

<SeqRecheck>, accessMtd=0x5576bdf656b0 <SeqNext>, node=0x5576c888b500) at
./build/../src/backend/executor/execScan.c:115

#2 ExecScan (node=0x5576c888b500, accessMtd=0x5576bdf656b0 <SeqNext>,

recheckMtd=0x5576bdf65740 <SeqRecheck>) at
./build/../src/backend/executor/execScan.c:200

#3 0x00005576bdf63f7d in ExecProcNode (node=0x5576c888b500) at

./build/../src/include/executor/executor.h:239

#4 ExecNestLoop (pstate=<optimized out>) at

./build/../src/backend/executor/nodeNestloop.c:109

#5 0x00005576bdf3e9a9 in ExecProcNode (node=0x5576c888b310) at

./build/../src/include/executor/executor.h:239

#6 EvalPlanQualNext (epqstate=epqstate@entry=0x5576bfbccfc0) at

./build/../src/backend/executor/execMain.c:2695

#7 0x00005576bdf3ee77 in EvalPlanQual

(epqstate=epqstate@entry=0x5576bfbccfc0,
relation=relation@entry=0x7fbfd57df168, rti=42,
inputslot=inputslot@entry=0x5576cfb66578) at ./build/../src/backend/>
executor/execMain.c:2467

#8 0x00005576bdf61f5e in ExecUpdate

(mtstate=mtstate@entry=0x5576bfbccec8, tupleid=0x7ffdd552f89a, oldtuple=0x0,
slot=<optimized out>, planSlot=planSlot@entry=0x5576c153f428,
epqstate=epqstate@entry=0x5576bfbccfc0, estate=0x5576bf5df080,

canSetTag=true) at

./build/../src/backend/executor/nodeModifyTable.c:1387

#9 0x00005576bdf62df9 in ExecModifyTable (pstate=0x5576bfbccec8) at

./build/../src/backend/executor/nodeModifyTable.c:2223

#10 0x00005576bdf3c23d in ExecProcNode (node=0x5576bfbccec8) at

./build/../src/include/executor/executor.h:239

#11 ExecutePlan (execute_once=<optimized out>, dest=0x5576be63ec00

<donothingDR>, direction=<optimized out>, numberTuples=0,
sendTuples=<optimized out>, operation=CMD_UPDATE,
use_parallel_mode=<optimized out>, planstate=0x5576bfbccec8,

estate=0x5576bf5df080) at

./build/../src/backend/executor/execMain.c:1646

#12 standard_ExecutorRun (queryDesc=0x5576bed15ae0, direction=<optimized

out>, count=0, execute_once=<optimized out>) at
./build/../src/backend/executor/execMain.c:364

#13 0x00007fc89b1a4125 in pgss_ExecutorRun (queryDesc=0x5576bed15ae0,

direction=ForwardScanDirection, count=0, execute_once=<optimized out>) at
./build/../contrib/pg_stat_statements/pg_stat_statements.c:893

#14 0x00005576be0993b5 in ProcessQuery (plan=<optimized out>,
sourceText=0x5576bed159d0 "UPDATE partitionedtable ia SET A=A, B=now()

FROM temptable tt WHERE ia.id=tt.ia_id AND A::text IS DISTINCT FROM
tt.A::text"...,

params=0x0, queryEnv=0x0, dest=0x5576be63ec00 <donothingDR>,

completionTag=0x7ffdd552ff40 "") at
./build/../src/backend/tcop/pquery.c:161

#15 0x00005576be0995f3 in PortalRunMulti

(portal=portal@entry=0x5576bea978f0, isTopLevel=isTopLevel@entry=true,
setHoldSnapshot=setHoldSnapshot@entry=false, dest=0x5576be63ec00
<donothingDR>, dest@entry=0x5576bea00890,

altdest=0x5576be63ec00 <donothingDR>, altdest@entry=0x5576bea00890,

completionTag=completionTag@entry=0x7ffdd552ff40 "") at
./build/../src/backend/tcop/pquery.c:1283

#16 0x00005576be09a1ff in PortalRun (portal=portal@entry=0x5576bea978f0,

count=count@entry=1, isTopLevel=isTopLevel@entry=true, run_once=<optimized
out>, dest=dest@entry=0x5576bea00890, altdest=altdest@entry=0x5576bea00890,

completionTag=0x7ffdd552ff40 "") at

./build/../src/backend/tcop/pquery.c:796

#17 0x00005576be097a31 in exec_execute_message (max_rows=1,

portal_name=0x5576bea00480 "") at
./build/../src/backend/tcop/postgres.c:2090

#18 PostgresMain (argc=<optimized out>, argv=argv@entry=0x5576bea2fbd0,

dbname=<optimized out>, username=<optimized out>) at
./build/../src/backend/tcop/postgres.c:4297

#19 0x00005576be01e53b in BackendRun (port=0x5576bea27dc0,

port=0x5576bea27dc0) at
./build/../src/backend/postmaster/postmaster.c:4437

#20 BackendStartup (port=0x5576bea27dc0) at

./build/../src/backend/postmaster/postmaster.c:4128

#21 ServerLoop () at ./build/../src/backend/postmaster/postmaster.c:1704
#22 0x00005576be01f513 in PostmasterMain (argc=5, argv=0x5576be9f9090) at

./build/../src/backend/postmaster/postmaster.c:1377

#23 0x00005576bdd95cf6 in main (argc=5, argv=0x5576be9f9090) at

./build/../src/backend/main/main.c:228

#0 EvalPlanQualFetchRowMark (epqstate=0x5576bfbccfc0, rti=2,

slot=slot@entry=0x5576c888b6c0) at
./build/../src/backend/executor/execMain.c:2590

earm = 0x5576c7176258
erm = 0x24
datum = <optimized out>
isNull = false
__func__ = "EvalPlanQualFetchRowMark"
#1 0x00005576bdf449ce in ExecScanFetch (recheckMtd=0x5576bdf65740

<SeqRecheck>, accessMtd=0x5576bdf656b0 <SeqNext>, node=0x5576c888b500) at
./build/../src/backend/executor/execScan.c:115

slot = 0x5576c888b6c0
epqstate = <optimized out>
scanrelid = <optimized out>
slot = <optimized out>
estate = 0x5576c888b100
estate = <optimized out>
epqstate = <optimized out>
scanrelid = <optimized out>
slot = <optimized out>
slot = <optimized out>
slot = <optimized out>
slot = <optimized out>
#2 ExecScan (node=0x5576c888b500,

Excerpt shows the pointer 0x24 of erm to be uninitialized'ish. Once i can
capture a second
stacktrace i might be able to confirm if this is indeed a random value or
always 0x24. However,
we're workarounding this by simply not running this query concurrently
anymore hence i might
lack new stacktraces soon.

UPDATE partitionedtable ia SET A=A, B=now() FROM temptable tt WHERE

ia.id=tt.ia_id AND A::text IS DISTINCT FROM tt.A::text;

QUERY PLAN

-------------------------------------------------------------------------------------------------------------------------------

Update on partitionedtable ia (cost=0.43..1202173.35 rows=465660

width=143)

Update on partitionedtable ia
Update on partitionedtable_0062 ia_1
Update on partitionedtable_0061 ia_2
Update on partitionedtable_0060 ia_3
[repeats for another ~400 Partitions]
[..]
-> Nested Loop (cost=0.43..3386.78 rows=1194 width=145)
-> Seq Scan on temptable tt (cost=0.00..22.00 rows=1200

width=46)

-> Index Scan using partitionedtable_pk on partitionedtable ia

(cost=0.43..2.79 rows=1 width=560)

Index Cond: (id = tt.ia_id)
Filter: ((document)::text IS DISTINCT FROM

(tt.document_json)::text)

-> Nested Loop (cost=0.42..3108.99 rows=1194 width=159)
-> Seq Scan on temptable tt (cost=0.00..22.00 rows=1200

width=46)

-> Index Scan using partitionedtable_062_pk on

partitionedtable_0062 ia_1 (cost=0.42..2.56 rows=1 width=518)

Index Cond: (id = tt.ia_id)
Filter: ((document)::text IS DISTINCT FROM

(tt.document_json)::text)

[repeats for another ~400 Partitions]
[..]

JIT:
Functions: 3510
Options: Inlining true, Optimization true, Expressions true, Deforming

true

(2344 rows)

The Query basically updates a massively partitioned* Table from the
contents
of a temptable.

*Inheritance-Partitioning on ID(not modified within the query)
*Inheritance has triggers. ISTM that they are not of any concern.

System is a current Ubuntu18.04 with a freshly pg_upgrade'd postgresql-12.1
pgdg-build,
including new statistics, indices and vacuum.

I'm currently trying to get this to a minimized and reproducible form as i
do not see an
obvious error in the code yet. For the time being i fail to actually
recreate as accurate as
production does :)

regards,
julian