From b2a9bb0be2a32e679089d343cf31d0c61f4162ec Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 19 Jun 2025 12:08:27 +0900 Subject: [PATCH v1 07/12] Add support for bigint TOAST values This commit adds the possibility to define TOAST tables with bigint as value ID, baesd on the GUC default_toast_type. All the external TOAST pointers still rely on varatt_external and a single vartag, with all the values inserted in the bigint TOAST tables fed from the existing OID value generator. This will be changed in an upcoming patch that adds more vartag_external types and its associated structures, with the code being able to use a different external TOAST pointer depending on the attribute type of chunk_id in TOAST relations. All the changes done here are mechanical, with all the TOAST code able to do chunk ID lookups based on the two types supported. XXX: Catalog version bump required. --- src/backend/access/common/toast_internals.c | 127 +++++++++++++++----- src/backend/access/heap/heaptoast.c | 20 ++- src/backend/catalog/toasting.c | 37 +++++- doc/src/sgml/storage.sgml | 7 +- contrib/amcheck/verify_heapam.c | 19 ++- 5 files changed, 166 insertions(+), 44 deletions(-) diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 4a1342da6e1b..5e28a33557dc 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -18,6 +18,7 @@ #include "access/heapam.h" #include "access/heaptoast.h" #include "access/table.h" +#include "access/toast_counter.h" #include "access/toast_internals.h" #include "access/xact.h" #include "catalog/catalog.h" @@ -143,6 +144,8 @@ toast_save_datum(Relation rel, Datum value, Pointer dval = DatumGetPointer(value); int num_indexes; int validIndex; + Oid toast_typid; + uint64 new_valueid = 0; Assert(!VARATT_IS_EXTERNAL(value)); @@ -154,6 +157,13 @@ toast_save_datum(Relation rel, Datum value, toastrel = table_open(rel->rd_rel->reltoastrelid, RowExclusiveLock); toasttupDesc = toastrel->rd_att; + /* + * This varies depending on the attribute of the toast relation used for + * its values. + */ + toast_typid = TupleDescAttr(toasttupDesc, 0)->atttypid; + Assert(toast_typid == OIDOID || toast_typid == INT8OID); + /* Open all the toast indexes and look for the valid one */ validIndex = toast_open_indexes(toastrel, RowExclusiveLock, @@ -212,29 +222,41 @@ toast_save_datum(Relation rel, Datum value, toast_pointer.va_toastrelid = RelationGetRelid(toastrel); /* - * Choose an OID to use as the value ID for this toast value. + * Choose a new value to use as the value ID for this toast value, be it + * for OID or int8-based TOAST relations. * - * Normally we just choose an unused OID within the toast table. But + * Normally we just choose an unused value within the toast table. But * during table-rewriting operations where we are preserving an existing - * toast table OID, we want to preserve toast value OIDs too. So, if + * toast table OID, we want to preserve toast value IDs too. So, if * rd_toastoid is set and we had a prior external value from that same * toast table, re-use its value ID. If we didn't have a prior external * value (which is a corner case, but possible if the table's attstorage * options have been changed), we have to pick a value ID that doesn't - * conflict with either new or existing toast value OIDs. + * conflict with either new or existing toast value IDs. If the TOAST + * table uses 8-byte value IDs, we should not really care much about + * that. */ if (!OidIsValid(rel->rd_toastoid)) { - /* normal case: just choose an unused OID */ - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); + /* normal case: just choose an unused ID */ + if (toast_typid == OIDOID) + new_valueid = GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + else if (toast_typid == INT8OID) + new_valueid = GetNewToastId(); + else + Assert(false); } else { /* rewrite case: check to see if value was in old toast table */ - toast_pointer.va_valueid = InvalidOid; + if (toast_typid == OIDOID) + new_valueid = InvalidOid; + else if (toast_typid == INT8OID) + new_valueid = InvalidToastId; + else + Assert(false); if (oldexternal != NULL) { struct varatt_external old_toast_pointer; @@ -242,10 +264,16 @@ toast_save_datum(Relation rel, Datum value, Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal)); /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal); + if (old_toast_pointer.va_toastrelid == rel->rd_toastoid) { - /* This value came from the old toast table; reuse its OID */ - toast_pointer.va_valueid = old_toast_pointer.va_valueid; + uint64 old_valueid; + + /* + * The old and new toast relations match, hence their type + * of value match as well. + */ + old_valueid = old_toast_pointer.va_valueid; /* * There is a corner case here: the table rewrite might have @@ -265,34 +293,48 @@ toast_save_datum(Relation rel, Datum value, * be reclaimed by VACUUM. */ if (toastrel_valueid_exists(toastrel, - toast_pointer.va_valueid)) + old_valueid)) { /* Match, so short-circuit the data storage loop below */ data_todo = 0; } + + /* + * The old and new toast relations match, hence their type + * of value match as well + */ + new_valueid = old_valueid; } } - if (toast_pointer.va_valueid == InvalidOid) + if ((new_valueid == InvalidToastId && toast_typid == INT8OID) || + (new_valueid == InvalidOid && toast_typid == OIDOID)) { /* - * new value; must choose an OID that doesn't conflict in either - * old or new toast table + * new value; must choose a value that doesn't conflict in either + * old or new toast table. */ do { - toast_pointer.va_valueid = - GetNewOidWithIndex(toastrel, - RelationGetRelid(toastidxs[validIndex]), - (AttrNumber) 1); - } while (toastid_valueid_exists(rel->rd_toastoid, - toast_pointer.va_valueid)); + if (toast_typid == OIDOID) + new_valueid = GetNewOidWithIndex(toastrel, + RelationGetRelid(toastidxs[validIndex]), + (AttrNumber) 1); + else if (toast_typid == INT8OID) + new_valueid = GetNewToastId(); + } while (toastid_valueid_exists(rel->rd_toastoid, new_valueid)); } } + /* Now set the new value */ + toast_pointer.va_valueid = new_valueid; + /* * Initialize constant parts of the tuple data */ - t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); + if (toast_typid == OIDOID) + t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); + else if (toast_typid == INT8OID) + t_values[0] = Int64GetDatum(toast_pointer.va_valueid); t_values[2] = PointerGetDatum(&chunk_data); t_isnull[0] = false; t_isnull[1] = false; @@ -393,6 +435,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) HeapTuple toasttup; int num_indexes; int validIndex; + Oid toast_typid; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) return; @@ -404,6 +447,8 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) * Open the toast relation and its indexes */ toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock); + toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; + Assert(toast_typid == OIDOID || toast_typid == INT8OID); /* Fetch valid relation used for process */ validIndex = toast_open_indexes(toastrel, @@ -414,10 +459,18 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative) /* * Setup a scan key to find chunks with matching va_valueid */ - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(toast_pointer.va_valueid)); + if (toast_typid == OIDOID) + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(toast_pointer.va_valueid)); + else if (toast_typid == INT8OID) + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_INT8EQ, + Int64GetDatum(toast_pointer.va_valueid)); + else + Assert(false); /* * Find all the chunks. (We don't actually care whether we see them in @@ -464,6 +517,7 @@ toastrel_valueid_exists(Relation toastrel, uint64 valueid) int num_indexes; int validIndex; Relation *toastidxs; + Oid toast_typid; /* Fetch a valid index relation */ validIndex = toast_open_indexes(toastrel, @@ -471,13 +525,24 @@ toastrel_valueid_exists(Relation toastrel, uint64 valueid) &toastidxs, &num_indexes); + toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; + Assert(toast_typid == OIDOID || toast_typid == INT8OID); + /* * Setup a scan key to find chunks with matching va_valueid */ - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); + if (toast_typid == OIDOID) + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(valueid)); + else if (toast_typid == INT8OID) + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_INT8EQ, + Int64GetDatum(valueid)); + else + Assert(false); /* * Is there any such chunk? diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index ae8d502ddcd3..f45be1d0d401 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -640,6 +640,7 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, int num_indexes; int validIndex; int32 max_chunk_size; + Oid toast_typid; /* Look for the valid index of toast relation */ validIndex = toast_open_indexes(toastrel, @@ -647,6 +648,9 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, &toastidxs, &num_indexes); + toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid; + Assert(toast_typid == OIDOID || toast_typid == INT8OID); + max_chunk_size = TOAST_MAX_CHUNK_SIZE; totalchunks = ((attrsize - 1) / max_chunk_size) + 1; @@ -655,10 +659,18 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize, Assert(endchunk <= totalchunks); /* Set up a scan key to fetch from the index. */ - ScanKeyInit(&toastkey[0], - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); + if (toast_typid == OIDOID) + ScanKeyInit(&toastkey[0], + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(valueid)); + else if (toast_typid == INT8OID) + ScanKeyInit(&toastkey[0], + (AttrNumber) 1, + BTEqualStrategyNumber, F_INT8EQ, + Int64GetDatum(valueid)); + else + Assert(false); /* * No additional condition if fetching all chunks. Otherwise, use an diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index e595cb61b375..3df83c9835d4 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -149,6 +149,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, int16 coloptions[2]; ObjectAddress baseobject, toastobject; + Oid toast_typid = InvalidOid; /* * Is it already toasted? @@ -204,11 +205,40 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, snprintf(toast_idxname, sizeof(toast_idxname), "pg_toast_%u_index", relOid); + /* + * Determine the type OID to use for the value. If OIDOldToast is + * defined, we need to rely on the existing table for the job because + * we do not want to create an inconsistent relation that would conflict + * with the parent and break the world. + */ + if (!OidIsValid(OIDOldToast)) + { + if (default_toast_type == TOAST_TYPE_OID) + toast_typid = OIDOID; + else if (default_toast_type == TOAST_TYPE_INT8) + toast_typid = INT8OID; + else + Assert(false); + } + else + { + HeapTuple tuple; + Form_pg_attribute atttoast; + + /* For the chunk_id type. */ + tuple = SearchSysCacheAttNum(OIDOldToast, 1); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", OIDOldToast); + atttoast = (Form_pg_attribute) GETSTRUCT(tuple); + toast_typid = atttoast->atttypid; + ReleaseSysCache(tuple); + } + /* this is pretty painful... need a tuple descriptor */ tupdesc = CreateTemplateTupleDesc(3); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "chunk_id", - OIDOID, + toast_typid, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "chunk_seq", @@ -316,7 +346,10 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, collationIds[0] = InvalidOid; collationIds[1] = InvalidOid; - opclassIds[0] = OID_BTREE_OPS_OID; + if (toast_typid == OIDOID) + opclassIds[0] = OID_BTREE_OPS_OID; + else if (toast_typid == INT8OID) + opclassIds[0] = INT8_BTREE_OPS_OID; opclassIds[1] = INT4_BTREE_OPS_OID; coloptions[0] = 0; diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 61250799ec07..1b2592645ab4 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -419,14 +419,15 @@ most TOAST_MAX_CHUNK_SIZE bytes (by default this value is chose so that four chunk rows will fit on a page, making it about 2000 bytes). Each chunk is stored as a separate row in the TOAST table belonging to the owning table. Every -TOAST table has the columns chunk_id (an OID -identifying the particular TOASTed value), +TOAST table has the columns +chunk_id (an OID or an 8-byte integer identifying +the particular TOASTed value), chunk_seq (a sequence number for the chunk within its value), and chunk_data (the actual data of the chunk). A unique index on chunk_id and chunk_seq provides fast retrieval of the values. A pointer datum representing an out-of-line on-disk TOASTed value therefore needs to store the OID of the -TOAST table in which to look and the OID of the specific value +TOAST table in which to look and the specific value (its chunk_id). For convenience, pointer datums also store the logical datum size (original uncompressed data length), physical stored size (different if compression was applied), and the compression method used, if diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 51ac416f7c6e..885400fa7058 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -1877,6 +1877,9 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) int32 last_chunk_seq; uint64 toast_valueid; int32 max_chunk_size = TOAST_MAX_CHUNK_SIZE; + Oid toast_typid; + + toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid; extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer); last_chunk_seq = (extsize - 1) / max_chunk_size; @@ -1884,10 +1887,18 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta) /* * Setup a scan key to find chunks in toast table with matching va_valueid */ - ScanKeyInit(&toastkey, - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(ta->toast_pointer.va_valueid)); + if (toast_typid == OIDOID) + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(ta->toast_pointer.va_valueid)); + else if (toast_typid == INT8OID) + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_INT8EQ, + Int64GetDatum(ta->toast_pointer.va_valueid)); + else + Assert(false); /* * Check if any chunks for this toasted object exist in the toast table, -- 2.49.0