From b1bf73d8111b0620558542c34098e0fee5d19b58 Mon Sep 17 00:00:00 2001 From: Sokolov Yura aka funny_falcon Date: Mon, 22 May 2017 11:25:51 +0300 Subject: [PATCH] attr storage SEEKABLE and EXTSEEKABLE EXTSEEKABLE is like EXTERNAL, but every chunk is compressed, so it is still seekable, but compressed. SEEKABLE is mix of EXTSEEKABLE and MAIN: if value is small, then it first compressed and could be saved inline then. --- src/backend/access/heap/tuptoaster.c | 78 ++++++++++++++++++++----- src/backend/commands/tablecmds.c | 8 +++ src/backend/commands/typecmds.c | 4 ++ src/backend/replication/logical/reorderbuffer.c | 9 +++ src/bin/pg_dump/pg_dump.c | 10 ++++ 5 files changed, 93 insertions(+), 16 deletions(-) diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index aa5a45d..76eac9d 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -69,7 +69,7 @@ typedef struct toast_compress_header static void toast_delete_datum(Relation rel, Datum value, bool is_speculative); static Datum toast_save_datum(Relation rel, Datum value, - struct varlena * oldexternal, int options); + struct varlena * oldexternal, int options, bool compress); static bool toastrel_valueid_exists(Relation toastrel, Oid valueid); static bool toastid_valueid_exists(Oid toastrelid, Oid valueid); static struct varlena *toast_fetch_datum(struct varlena * attr); @@ -734,7 +734,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, /* * Look for attributes with attstorage 'x' to compress. Also find large - * attributes with attstorage 'x' or 'e', and store them external. + * attributes with attstorage 'x', 'e', 'z' or 's', and store them external. */ while (heap_compute_data_size(tupleDesc, toast_values, toast_isnull) > maxDataLen) @@ -755,7 +755,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, continue; /* can't happen, toast_action would be 'p' */ if (VARATT_IS_COMPRESSED(DatumGetPointer(toast_values[i]))) continue; - if (att[i]->attstorage != 'x' && att[i]->attstorage != 'e') + if (att[i]->attstorage != 'x' && att[i]->attstorage != 'e' && + att[i]->attstorage != 'z' && att[i]->attstorage != 's') continue; if (toast_sizes[i] > biggest_size) { @@ -795,7 +796,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, } else { - /* has attstorage 'e', ignore on subsequent compression passes */ + /* has attstorage 'e' or 'z', ignore on subsequent compression passes */ toast_action[i] = 'x'; } @@ -813,7 +814,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, old_value = toast_values[i]; toast_action[i] = 'p'; toast_values[i] = toast_save_datum(rel, toast_values[i], - toast_oldexternal[i], options); + toast_oldexternal[i], options, + att[i]->attstorage == 'z' || + att[i]->attstorage == 's'); if (toast_free[i]) pfree(DatumGetPointer(old_value)); toast_free[i] = true; @@ -823,7 +826,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, } /* - * Second we look for attributes of attstorage 'x' or 'e' that are still + * Second we look for attributes of attstorage 'x', 'e' or 'z' that are still * inline. But skip this if there's no toast table to push them to. */ while (heap_compute_data_size(tupleDesc, @@ -845,7 +848,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, continue; if (VARATT_IS_EXTERNAL(DatumGetPointer(toast_values[i]))) continue; /* can't happen, toast_action would be 'p' */ - if (att[i]->attstorage != 'x' && att[i]->attstorage != 'e') + if (att[i]->attstorage != 'x' && att[i]->attstorage != 'e' && + att[i]->attstorage != 'z') continue; if (toast_sizes[i] > biggest_size) { @@ -864,7 +868,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, old_value = toast_values[i]; toast_action[i] = 'p'; toast_values[i] = toast_save_datum(rel, toast_values[i], - toast_oldexternal[i], options); + toast_oldexternal[i], options, + att[i]->attstorage == 'z'); if (toast_free[i]) pfree(DatumGetPointer(old_value)); toast_free[i] = true; @@ -874,7 +879,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, } /* - * Round 3 - this time we take attributes with storage 'm' into + * Round 3 - this time we take attributes with storage 'm' or 's' into * compression */ while (heap_compute_data_size(tupleDesc, @@ -896,7 +901,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, continue; /* can't happen, toast_action would be 'p' */ if (VARATT_IS_COMPRESSED(DatumGetPointer(toast_values[i]))) continue; - if (att[i]->attstorage != 'm') + if (att[i]->attstorage != 'm' && att[i]->attstorage != 's') continue; if (toast_sizes[i] > biggest_size) { @@ -934,9 +939,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, } /* - * Finally we store attributes of type 'm' externally. At this point we - * increase the target tuple size, so that 'm' attributes aren't stored - * externally unless really necessary. + * Finally we store attributes of type 'm' or 's' externally. At this + * point we increase the target tuple size, so that 'm' or 's' attributes + * aren't stored externally unless really necessary. */ maxDataLen = TOAST_TUPLE_TARGET_MAIN - hoff; @@ -959,7 +964,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, continue; if (VARATT_IS_EXTERNAL(DatumGetPointer(toast_values[i]))) continue; /* can't happen, toast_action would be 'p' */ - if (att[i]->attstorage != 'm') + if (att[i]->attstorage != 'm' && att[i]->attstorage != 's') continue; if (toast_sizes[i] > biggest_size) { @@ -978,7 +983,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, old_value = toast_values[i]; toast_action[i] = 'p'; toast_values[i] = toast_save_datum(rel, toast_values[i], - toast_oldexternal[i], options); + toast_oldexternal[i], options, + false); if (toast_free[i]) pfree(DatumGetPointer(old_value)); toast_free[i] = true; @@ -1468,7 +1474,7 @@ toast_get_valid_index(Oid toastoid, LOCKMODE lock) */ static Datum toast_save_datum(Relation rel, Datum value, - struct varlena * oldexternal, int options) + struct varlena * oldexternal, int options, bool compress) { Relation toastrel; Relation *toastidxs; @@ -1664,7 +1670,19 @@ toast_save_datum(Relation rel, Datum value, t_values[1] = Int32GetDatum(chunk_seq++); SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ); memcpy(VARDATA(&chunk_data), data_p, chunk_size); + if (compress) + { + Datum compressed; + compressed = toast_compress_datum(t_values[2]); + if (compressed != PointerGetDatum(NULL)) + t_values[2] = compressed; + } toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull); + if (compress && t_values[2] != PointerGetDatum(&chunk_data)) + { + pfree(DatumGetPointer(t_values[2])); + t_values[2] = PointerGetDatum(&chunk_data); + } heap_insert(toastrel, toasttup, mycid, options, NULL); @@ -1888,6 +1906,7 @@ toast_fetch_datum(struct varlena * attr) nextidx; int32 numchunks; Pointer chunk; + struct varlena *decompressed; bool isnull; char *chunkdata; int32 chunksize; @@ -1952,6 +1971,7 @@ toast_fetch_datum(struct varlena * attr) Assert(!isnull); chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); Assert(!isnull); + decompressed = NULL; if (!VARATT_IS_EXTENDED(chunk)) { chunksize = VARSIZE(chunk) - VARHDRSZ; @@ -1963,6 +1983,12 @@ toast_fetch_datum(struct varlena * attr) chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; chunkdata = VARDATA_SHORT(chunk); } + else if (VARATT_IS_COMPRESSED(chunk)) + { + decompressed = toast_decompress_datum((struct varlena*)chunk); + chunksize = VARSIZE(decompressed) - VARHDRSZ; + chunkdata = VARDATA(decompressed); + } else { /* should never happen */ @@ -2014,6 +2040,12 @@ toast_fetch_datum(struct varlena * attr) chunkdata, chunksize); + if (decompressed != NULL) + { + pfree(decompressed); + decompressed = NULL; + } + nextidx++; } @@ -2065,6 +2097,7 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length) int32 endoffset; int totalchunks; Pointer chunk; + struct varlena *decompressed; bool isnull; char *chunkdata; int32 chunksize; @@ -2178,6 +2211,7 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length) Assert(!isnull); chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); Assert(!isnull); + decompressed = NULL; if (!VARATT_IS_EXTENDED(chunk)) { chunksize = VARSIZE(chunk) - VARHDRSZ; @@ -2189,6 +2223,12 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length) chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; chunkdata = VARDATA_SHORT(chunk); } + else if (VARATT_IS_COMPRESSED(chunk)) + { + decompressed = toast_decompress_datum((struct varlena*)chunk); + chunksize = VARSIZE(decompressed) - VARHDRSZ; + chunkdata = VARDATA(decompressed); + } else { /* should never happen */ @@ -2248,6 +2288,12 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length) chunkdata + chcpystrt, (chcpyend - chcpystrt) + 1); + if (decompressed != NULL) + { + pfree(decompressed); + decompressed = NULL; + } + nextidx++; } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index fb961e4..e2b9e95 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1530,10 +1530,14 @@ storage_name(char c) return "PLAIN"; case 'm': return "MAIN"; + case 's': + return "SEEKABLE"; case 'x': return "EXTENDED"; case 'e': return "EXTERNAL"; + case 'z': + return "EXTSEEKABLE"; default: return "???"; } @@ -6277,10 +6281,14 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc newstorage = 'p'; else if (pg_strcasecmp(storagemode, "external") == 0) newstorage = 'e'; + else if (pg_strcasecmp(storagemode, "extseekable") == 0) + newstorage = 'z'; else if (pg_strcasecmp(storagemode, "extended") == 0) newstorage = 'x'; else if (pg_strcasecmp(storagemode, "main") == 0) newstorage = 'm'; + else if (pg_strcasecmp(storagemode, "seekable") == 0) + newstorage = 's'; else { ereport(ERROR, diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index c765e97..15bdcfb 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -404,10 +404,14 @@ DefineType(ParseState *pstate, List *names, List *parameters) storage = 'p'; else if (pg_strcasecmp(a, "external") == 0) storage = 'e'; + else if (pg_strcasecmp(a, "extseekable") == 0) + storage = 'z'; else if (pg_strcasecmp(a, "extended") == 0) storage = 'x'; else if (pg_strcasecmp(a, "main") == 0) storage = 'm'; + else if (pg_strcasecmp(a, "seekable") == 0) + storage = 's'; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 524946a..803e79a 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -2868,6 +2868,15 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, ctup = cchange->data.tp.newtuple; chunk = DatumGetPointer( fastgetattr(&ctup->tuple, 3, toast_desc, &isnull)); + if (VARATT_IS_COMPRESSED(chunk)) + { + struct varlena *tmp = chunk; + + /* indirect call to toast_decompress_datum */ + chunk = heap_tuple_untoast_attr(tmp); + if (tmp != chunk) + pfree(tmp); + } Assert(!isnull); Assert(!VARATT_IS_EXTERNAL(chunk)); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 053ae0e..85f1a53 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -10295,10 +10295,14 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) appendPQExpBufferStr(q, ",\n STORAGE = plain"); else if (strcmp(typstorage, "e") == 0) appendPQExpBufferStr(q, ",\n STORAGE = external"); + else if (strcmp(typstorage, "z") == 0) + appendPQExpBufferStr(q, ",\n STORAGE = extseekable"); else if (strcmp(typstorage, "x") == 0) appendPQExpBufferStr(q, ",\n STORAGE = extended"); else if (strcmp(typstorage, "m") == 0) appendPQExpBufferStr(q, ",\n STORAGE = main"); + else if (strcmp(typstorage, "s") == 0) + appendPQExpBufferStr(q, ",\n STORAGE = seekable"); if (strcmp(typbyval, "t") == 0) appendPQExpBufferStr(q, ",\n PASSEDBYVALUE"); @@ -15608,9 +15612,15 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) case 'e': storage = "EXTERNAL"; break; + case 'z': + storage = "EXTSEEKABLE"; + break; case 'm': storage = "MAIN"; break; + case 's': + storage = "SEEKABLE"; + break; case 'x': storage = "EXTENDED"; break; -- 2.9.3