Alternative variable length structure
Hi Hackers,
PostgreSQL can treat variable-length data flexibly, but therefore
it consumes more spaces if we store short data. Headers of
variable-length types use 4 bytes regardless of the data length.
My idea is to change the header itself to variable-length.
In order to reduce the size of short data, I wrote a patch to encode
lengths into the first several bits of structure. Also, the alignments
of the types were changed to 'char' from 'int'.
I know my patch is still insufficient, for example, the types cannot
be TOASTed. But I guess this compression works well for short text.
I'll appreciate any comments.
thanks.
---- the result of patch ----
# create table txttbl (v1 text, v2 text, v3 text, v4 text);
# create table strtbl (v1 string, v2 string, v3 string, v4 string);
# insert into txttbl values('A', 'B', 'C', 'D');
# insert into strtbl values('A', 'B', 'C', 'D');
# select * from pgstattuple('txttbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 57 <-- 28 + (5+3) + (5+3) + (5+3) + (5)
...
# select * from pgstattuple('strtbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 36 <-- 28 + 2 + 2 + 2 + 2
...
---
ITAGAKI Takahiro
NTT Cyber Space Laboratories
Attachments:
varlena2.patchapplication/octet-stream; name=varlena2.patchDownload
diff -cpr pgsql-orig/src/backend/access/common/heaptuple.c pgsql/src/backend/access/common/heaptuple.c
*** pgsql-orig/src/backend/access/common/heaptuple.c 2005-09-08 11:10:16.253332616 +0900
--- pgsql/src/backend/access/common/heaptuple.c 2005-09-08 11:11:04.884939488 +0900
*************** heap_fill_tuple(TupleDesc tupleDesc,
*** 173,178 ****
--- 173,185 ----
data_length = strlen(DatumGetCString(values[i])) + 1;
memcpy(data, DatumGetPointer(values[i]), data_length);
}
+ else if (att[i]->attlen == -3)
+ {
+ /* varlena2 */
+ *infomask |= HEAP_HASVARWIDTH;
+ data_length = varlena2size(DatumGetRawVarlena2P(values[i]));
+ memcpy(data, DatumGetPointer(values[i]), data_length);
+ }
else
{
/* fixed-length pass-by-reference */
*************** DataFill(char *data,
*** 272,277 ****
--- 279,291 ----
data_length = strlen(DatumGetCString(values[i])) + 1;
memcpy(data, DatumGetPointer(values[i]), data_length);
}
+ else if (att[i]->attlen == -3)
+ {
+ /* varlena2 */
+ *infomask |= HEAP_HASVARWIDTH;
+ data_length = varlena2size(DatumGetRawVarlena2P(values[i]));
+ memcpy(data, DatumGetPointer(values[i]), data_length);
+ }
else
{
/* fixed-length pass-by-reference */
diff -cpr pgsql-orig/src/backend/catalog/pg_type.c pgsql/src/backend/catalog/pg_type.c
*** pgsql-orig/src/backend/catalog/pg_type.c 2005-09-08 11:10:16.343318936 +0900
--- pgsql/src/backend/catalog/pg_type.c 2005-09-08 11:11:04.885939336 +0900
*************** TypeCreate(const char *typeName,
*** 189,195 ****
*/
if (!(internalSize > 0 ||
internalSize == -1 ||
! internalSize == -2))
ereport(ERROR,
(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
errmsg("invalid type internal size %d",
--- 189,196 ----
*/
if (!(internalSize > 0 ||
internalSize == -1 ||
! internalSize == -2 ||
! internalSize == -3))
ereport(ERROR,
(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
errmsg("invalid type internal size %d",
diff -cpr pgsql-orig/src/backend/utils/adt/arrayfuncs.c pgsql/src/backend/utils/adt/arrayfuncs.c
*** pgsql-orig/src/backend/utils/adt/arrayfuncs.c 2005-09-08 11:10:16.217338088 +0900
--- pgsql/src/backend/utils/adt/arrayfuncs.c 2005-09-08 11:11:04.888938880 +0900
*************** ReadArrayStr(char *arrayStr,
*** 817,831 ****
totbytes += sizeof(int32);
totbytes = att_align(totbytes, typalign);
}
! else
{
/* dummy cstring value */
- Assert(typlen == -2);
values[i] = PointerGetDatum(palloc(1));
*((char *) DatumGetPointer(values[i])) = '\0';
totbytes += 1;
totbytes = att_align(totbytes, typalign);
}
}
}
*nbytes = totbytes;
--- 817,839 ----
totbytes += sizeof(int32);
totbytes = att_align(totbytes, typalign);
}
! else if (typlen == -2)
{
/* dummy cstring value */
values[i] = PointerGetDatum(palloc(1));
*((char *) DatumGetPointer(values[i])) = '\0';
totbytes += 1;
totbytes = att_align(totbytes, typalign);
}
+ else
+ {
+ /* dummy varlena2 value */
+ Assert(typlen == -3);
+ values[i] = PointerGetDatum(palloc(1));
+ *((char *) DatumGetPointer(values[i])) = 0;
+ totbytes += 1;
+ totbytes = att_align(totbytes, typalign);
+ }
}
}
*nbytes = totbytes;
diff -cpr pgsql-orig/src/backend/utils/adt/datum.c pgsql/src/backend/utils/adt/datum.c
*** pgsql-orig/src/backend/utils/adt/datum.c 2005-09-08 11:10:16.201340520 +0900
--- pgsql/src/backend/utils/adt/datum.c 2005-09-08 11:11:04.889938728 +0900
*************** datumGetSize(Datum value, bool typByVal,
*** 93,98 ****
--- 93,110 ----
size = (Size) (strlen(s) + 1);
}
+ else if (typLen == -3)
+ {
+ /* It is a varlena2 datatype */
+ struct varlena2 *s = (struct varlena2 *) DatumGetPointer(value);
+
+ if (!PointerIsValid(s))
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_EXCEPTION),
+ errmsg("invalid Datum pointer")));
+
+ size = (Size) varlena2size(s);
+ }
else
{
elog(ERROR, "invalid typLen: %d", typLen);
diff -cpr pgsql-orig/src/backend/utils/adt/varlena.c pgsql/src/backend/utils/adt/varlena.c
*** pgsql-orig/src/backend/utils/adt/varlena.c 2005-09-08 11:10:16.210339152 +0900
--- pgsql/src/backend/utils/adt/varlena.c 2005-09-08 11:44:36.785084176 +0900
*************** pg_column_size(PG_FUNCTION_ARGS)
*** 2684,2689 ****
--- 2684,2694 ----
/* cstring */
result = strlen(DatumGetCString(value)) + 1;
}
+ else if (typlen == -3)
+ {
+ /* varlena2 type */
+ result = varlena2size(DatumGetRawVarlena2P(value));
+ }
else
{
/* ordinary fixed-width type */
*************** pg_column_size(PG_FUNCTION_ARGS)
*** 2692,2694 ****
--- 2697,2951 ----
PG_RETURN_INT32(result);
}
+
+ /* varlena2 */
+
+ /*
+ * The length of varlena2 is encoded as follows:
+ *
+ * | First | Trailing | Total | Max |
+ * | byte | bytes | bits | length |
+ * +----------+----------+-------+---------+
+ * | 0******* | 0 | 7 | 127 |
+ * | 10****** | 1 | 14 | 16K -1 |
+ * | 110***** | 2 | 21 | 2M -1 |
+ * | 1110**** | 3 | 28 | 256M -1 |
+ * | 1111---- | 4 | 32 | 4G -1 |
+ */
+
+ static const uint8 VARLENA2HDRMAP[16] =
+ {
+ 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5,
+ };
+
+ /* Return the header size of a varlena2 */
+ #define VARLENA2HDRSZ(n) VARLENA2HDRMAP[(uint8)n[0] >> 4]
+
+ /* Decode the byte size of a varlena2 */
+ #define VARLENA2LENGTH_1(n) \
+ ((uint32)n[0])
+ #define VARLENA2LENGTH_2(n) \
+ (((uint32)(n[0] & 0x3F) << 8) \
+ + n[1])
+ #define VARLENA2LENGTH_3(n) \
+ (((uint32)(n[0] & 0x1F) << 16) \
+ + ((uint32)n[1] << 8) \
+ + n[2])
+ #define VARLENA2LENGTH_4(n) \
+ (((uint32)(n[0] & 0x0F) << 24) \
+ + ((uint32)n[1] << 16) \
+ + ((uint32)n[2] << 8) \
+ + n[3])
+ #define VARLENA2LENGTH_5(n) \
+ ( ((uint32)n[1] << 24) \
+ + ((uint32)n[2] << 16) \
+ + ((uint32)n[3] << 8) \
+ + n[4])
+
+
+ /* Return the total byte size of a varlena2 */
+ int varlena2size(const struct varlena2 *v)
+ {
+ const uint8* len = v->vl2_len;
+ switch(VARLENA2HDRSZ(len))
+ {
+ case 1: return 1 + VARLENA2LENGTH_1(len);
+ case 2: return 2 + VARLENA2LENGTH_2(len);
+ case 3: return 3 + VARLENA2LENGTH_3(len);
+ case 4: return 4 + VARLENA2LENGTH_4(len);
+ case 5: return 5 + VARLENA2LENGTH_5(len);
+ default:
+ elog(ERROR, "varlena2size");
+ return -1;
+ }
+ }
+
+ /* Return the data buffer and the data length of a varlena2 */
+ static char *varlena2extract(struct varlena2 *v, int* length)
+ {
+ const uint8* len = v->vl2_len;
+ switch(VARLENA2HDRSZ(len))
+ {
+ case 1:
+ *length = VARLENA2LENGTH_1(len);
+ return (char *) v + 1;
+ case 2:
+ *length = VARLENA2LENGTH_2(len);
+ return (char *) v + 2;
+ case 3:
+ *length = VARLENA2LENGTH_3(len);
+ return (char *) v + 3;
+ case 4:
+ *length = VARLENA2LENGTH_4(len);
+ return (char *) v + 4;
+ case 5:
+ *length = VARLENA2LENGTH_5(len);
+ return (char *) v + 5;
+ default:
+ elog(ERROR, "varlena2extract");
+ return NULL;
+ }
+ }
+
+ /* Return a new varlena2. */
+ static struct varlena2 *varlena2new(int length, char** data)
+ {
+ struct varlena2 *v;
+ if(length < (1 << 7))
+ {
+ v = (struct varlena2*) palloc(length + 1);
+ v->vl2_len[0] = (uint8)length;
+ *data = (char *) v + 1;
+ }
+ else if(length < (1 << 14))
+ {
+ v = (struct varlena2*) palloc(length + 2);
+ v->vl2_len[0] = (uint8)((length & 0x3F00) >> 8) | 0x80;
+ v->vl2_len[1] = (uint8)((length & 0x00FF));
+ *data = (char *) v + 2;
+ }
+ else if(length < (1 << 21))
+ {
+ v = (struct varlena2*) palloc(length + 3);
+ v->vl2_len[0] = (uint8)((length & 0x1F0000) >> 16) | 0xC0;
+ v->vl2_len[1] = (uint8)((length & 0x00FF00) >> 8);
+ v->vl2_len[2] = (uint8)((length & 0x0000FF));
+ *data = (char *) v + 3;
+ }
+ else if(length < (1 << 28))
+ {
+ v = (struct varlena2*) palloc(length + 4);
+ v->vl2_len[0] = (uint8)((length & 0x0F000000) >> 24) | 0xE0;
+ v->vl2_len[1] = (uint8)((length & 0x00FF0000) >> 16);
+ v->vl2_len[2] = (uint8)((length & 0x0000FF00) >> 8);
+ v->vl2_len[3] = (uint8)((length & 0x000000FF));
+ *data = (char *) v + 4;
+ }
+ else
+ {
+ v = (struct varlena2*) palloc(length + 5);
+ v->vl2_len[0] = 0xF0;
+ v->vl2_len[1] = (uint8)((length & 0xFF000000) >> 24);
+ v->vl2_len[2] = (uint8)((length & 0x00FF0000) >> 16);
+ v->vl2_len[3] = (uint8)((length & 0x0000FF00) >> 8);
+ v->vl2_len[4] = (uint8)((length & 0x000000FF));
+ *data = (char *) v + 5;
+ }
+ return v;
+ }
+
+ #define StringNew(length, data) ((String *) varlena2new((length), (data)))
+ #define StringExtract varlena2extract
+
+ #define BinaryNew(length, data) ((Binary *) varlena2new((length), (data)))
+ #define BinaryExtract varlena2extract
+
+ /*
+ * stringin - converts "..." to internal representation
+ */
+ Datum
+ stringin(PG_FUNCTION_ARGS)
+ {
+ char *inputText = PG_GETARG_CSTRING(0);
+ String *result;
+ char *data;
+ int len;
+
+ /* verify encoding */
+ len = strlen(inputText);
+ pg_verifymbstr(inputText, len, false);
+
+ result = StringNew(len, &data);
+ memcpy(data, inputText, len);
+
+ PG_RETURN_STRING_P(result);
+ }
+
+ /*
+ * stringout - converts internal representation to "..."
+ */
+ Datum
+ stringout(PG_FUNCTION_ARGS)
+ {
+ String *t = PG_GETARG_STRING_P(0);
+ int len;
+ char *data;
+ char *result;
+
+ data = StringExtract(t, &len);
+ result = (char *) palloc(len + 1);
+ memcpy(result, data, len);
+ result[len] = '\0';
+
+ PG_RETURN_CSTRING(result);
+ }
+
+ /*
+ * stringrecv - converts external binary format to string
+ */
+ Datum
+ stringrecv(PG_FUNCTION_ARGS)
+ {
+ StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
+ String *result;
+ char *data;
+ char *inputText;
+ int nbytes;
+
+ inputText = pq_getmsgtext(buf, buf->len - buf->cursor, &nbytes);
+
+ /* verify encoding */
+ pg_verifymbstr(inputText, nbytes, false);
+
+ result = StringNew(nbytes, &data);
+ memcpy(data, inputText, nbytes);
+ pfree(inputText);
+
+ PG_RETURN_STRING_P(result);
+ }
+
+ /*
+ * stringsend - converts string to binary format
+ */
+ Datum
+ stringsend(PG_FUNCTION_ARGS)
+ {
+ String *t = PG_GETARG_STRING_P(0);
+ StringInfoData buf;
+ char *data;
+ int len;
+
+ data = StringExtract(t, &len);
+
+ pq_begintypsend(&buf);
+ pq_sendtext(&buf, data, len);
+ PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
+ }
+
+ Datum
+ binaryin(PG_FUNCTION_ARGS)
+ {
+ elog(ERROR, "binaryin not implemented");
+ PG_RETURN_VOID();
+ }
+
+ Datum
+ binaryout(PG_FUNCTION_ARGS)
+ {
+ elog(ERROR, "binaryout not implemented");
+ PG_RETURN_VOID();
+ }
+
+ Datum
+ binaryrecv(PG_FUNCTION_ARGS)
+ {
+ elog(ERROR, "binaryrecv not implemented");
+ PG_RETURN_VOID();
+ }
+
+ Datum
+ binarysend(PG_FUNCTION_ARGS)
+ {
+ elog(ERROR, "binarysend not implemented");
+ PG_RETURN_VOID();
+ }
diff -cpr pgsql-orig/src/backend/utils/fmgr/fmgr.c pgsql/src/backend/utils/fmgr/fmgr.c
*** pgsql-orig/src/backend/utils/fmgr/fmgr.c 2005-09-08 11:10:16.221337480 +0900
--- pgsql/src/backend/utils/fmgr/fmgr.c 2005-09-08 11:11:04.894937968 +0900
*************** pg_detoast_datum_slice(struct varlena *
*** 1828,1833 ****
--- 1828,1851 ----
}
/*-------------------------------------------------------------------------
+ * Support routines for toastable datatypes (varlena2)
+ *-------------------------------------------------------------------------
+ */
+
+ struct varlena2 *
+ pg_detoast_datum2(struct varlena2 * datum)
+ {
+ #if 1
+ return datum;
+ #else /* Not Implementd */
+ if (VARATT2_IS_EXTENDED(datum))
+ return (struct varlena2 *) heap_tuple_untoast_attr2((varattrib *) datum);
+ else
+ return datum;
+ #endif
+ }
+
+ /*-------------------------------------------------------------------------
* Support routines for extracting info from fn_expr parse tree
*
* These are needed by polymorphic functions, which accept multiple possible
diff -cpr pgsql-orig/src/include/access/tupmacs.h pgsql/src/include/access/tupmacs.h
*** pgsql-orig/src/include/access/tupmacs.h 2005-09-08 11:10:16.397310728 +0900
--- pgsql/src/include/access/tupmacs.h 2005-09-08 11:11:04.895937816 +0900
***************
*** 121,131 ****
( \
(cur_offset) + VARATT_SIZE(DatumGetPointer(attval)) \
) \
! : \
( \
- AssertMacro((attlen) == -2), \
(cur_offset) + (strlen(DatumGetCString(attval)) + 1) \
! )) \
)
/*
--- 121,135 ----
( \
(cur_offset) + VARATT_SIZE(DatumGetPointer(attval)) \
) \
! : (((attlen) == -2) ? \
( \
(cur_offset) + (strlen(DatumGetCString(attval)) + 1) \
! ) \
! : \
! ( \
! AssertMacro((attlen) == -3), \
! (cur_offset) + varlena2size(DatumGetRawVarlena2P(attval)) \
! ))) \
)
/*
diff -cpr pgsql-orig/src/include/c.h pgsql/src/include/c.h
*** pgsql-orig/src/include/c.h 2005-09-08 11:10:16.361316200 +0900
--- pgsql/src/include/c.h 2005-09-08 11:11:04.896937664 +0900
*************** typedef struct varlena text;
*** 431,436 ****
--- 431,451 ----
typedef struct varlena BpChar; /* blank-padded char, ie SQL char(n) */
typedef struct varlena VarChar; /* var-length char, ie SQL varchar(n) */
+ /* ----------------
+ * Variable-length V2 datatypes all share the 'struct varlena2' header.
+ * ----------------
+ */
+ struct varlena2
+ {
+ uint8 vl2_len[1];
+ /* vl2_data[] */
+ };
+
+ extern int varlena2size(const struct varlena2 *v);
+
+ typedef struct varlena2 Binary;
+ typedef struct varlena2 String;
+
/*
* Specialized array types. These are physically laid out just the same
* as regular arrays (so that the regular array subscripting code works
diff -cpr pgsql-orig/src/include/catalog/pg_proc.h pgsql/src/include/catalog/pg_proc.h
*** pgsql-orig/src/include/catalog/pg_proc.h 2005-09-08 11:10:16.446303280 +0900
--- pgsql/src/include/catalog/pg_proc.h 2005-09-08 11:11:04.902936752 +0900
*************** DESCR("GiST support");
*** 3763,3768 ****
--- 3763,3785 ----
DATA(insert OID = 2592 ( gist_circle_compress PGNSP PGUID 12 f f t f i 1 2281 "2281" _null_ _null_ _null_ gist_circle_compress - _null_ ));
DESCR("GiST support");
+ DATA(insert OID = 2593 ( stringin PGNSP PGUID 12 f f t f i 1 25 "2275" _null_ _null_ _null_ stringin - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2594 ( stringout PGNSP PGUID 12 f f t f i 1 2275 "31" _null_ _null_ _null_ stringout - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2595 ( stringrecv PGNSP PGUID 12 f f t f s 1 25 "2281" _null_ _null_ _null_ stringrecv - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2596 ( stringsend PGNSP PGUID 12 f f t f s 1 17 "31" _null_ _null_ _null_ stringsend - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2597 ( binaryin PGNSP PGUID 12 f f t f i 1 25 "2275" _null_ _null_ _null_ binaryin - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2598 ( binaryout PGNSP PGUID 12 f f t f i 1 2275 "21" _null_ _null_ _null_ binaryout - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2599 ( binaryrecv PGNSP PGUID 12 f f t f s 1 25 "2281" _null_ _null_ _null_ binaryrecv - _null_ ));
+ DESCR("I/O");
+ DATA(insert OID = 2600 ( binarysend PGNSP PGUID 12 f f t f s 1 17 "32" _null_ _null_ _null_ binarysend - _null_ ));
+ DESCR("I/O");
+
/*
* Symbolic values for provolatile column: these indicate whether the result
diff -cpr pgsql-orig/src/include/catalog/pg_type.h pgsql/src/include/catalog/pg_type.h
*** pgsql-orig/src/include/catalog/pg_type.h 2005-09-08 11:10:16.447303128 +0900
--- pgsql/src/include/catalog/pg_type.h 2005-09-08 11:11:04.903936600 +0900
*************** DATA(insert OID = 30 ( oidvector PGNSP
*** 306,311 ****
--- 306,319 ----
DESCR("array of oids, used in system tables");
#define OIDVECTOROID 30
+ DATA(insert OID = 31 ( string PGNSP PGUID -3 f b t \054 0 0 stringin stringout stringrecv stringsend - c p f 0 -1 0 _null_ _null_ ));
+ DESCR("variable-length string, no limit specified");
+ #define STRINGOID 31
+
+ DATA(insert OID = 32 ( binary PGNSP PGUID -3 f b t \054 0 0 binaryin binaryout binaryrecv binarysend - c p f 0 -1 0 _null_ _null_ ));
+ DESCR("variable-length string, binary values escaped");
+ #define BINARYOID 32
+
/* hand-built rowtype entries for bootstrapped catalogs: */
DATA(insert OID = 71 ( pg_type PGNSP PGUID -1 f c t \054 1247 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_ ));
diff -cpr pgsql-orig/src/include/fmgr.h pgsql/src/include/fmgr.h
*** pgsql-orig/src/include/fmgr.h 2005-09-08 11:10:16.418307536 +0900
--- pgsql/src/include/fmgr.h 2005-09-08 11:11:04.904936448 +0900
*************** extern struct varlena *pg_detoast_datum_
*** 171,176 ****
--- 171,186 ----
(int32) f, (int32) c)
/*
+ * Support for fetching detoasted copies of toastable varlena2 datatypes.
+ * pg_detoast_datum2() gives you either the input datum (if not toasted)
+ * or a detoasted copy allocated with palloc().
+ */
+ extern struct varlena2 *pg_detoast_datum2(struct varlena2 * datum);
+
+ #define PG_DETOAST_DATUM2(datum) \
+ pg_detoast_datum2((struct varlena2 *) DatumGetPointer(datum))
+
+ /*
* Support for cleaning up detoasted copies of inputs. This must only
* be used for pass-by-ref datatypes, and normally would only be used
* for toastable types. If the given pointer is different from the
*************** extern struct varlena *pg_detoast_datum_
*** 204,217 ****
--- 214,234 ----
#define PG_GETARG_INT64(n) DatumGetInt64(PG_GETARG_DATUM(n))
/* use this if you want the raw, possibly-toasted input datum: */
#define PG_GETARG_RAW_VARLENA_P(n) ((struct varlena *) PG_GETARG_POINTER(n))
+ #define PG_GETARG_RAW_VARLENA2_P(n) ((struct varlena2 *) PG_GETARG_POINTER(n))
/* use this if you want the input datum de-toasted: */
#define PG_GETARG_VARLENA_P(n) PG_DETOAST_DATUM(PG_GETARG_DATUM(n))
+ #define PG_GETARG_VARLENA2_P(n) PG_DETOAST_DATUM2(PG_GETARG_DATUM(n))
/* DatumGetFoo macros for varlena types will typically look like this: */
#define DatumGetByteaP(X) ((bytea *) PG_DETOAST_DATUM(X))
#define DatumGetTextP(X) ((text *) PG_DETOAST_DATUM(X))
#define DatumGetBpCharP(X) ((BpChar *) PG_DETOAST_DATUM(X))
#define DatumGetVarCharP(X) ((VarChar *) PG_DETOAST_DATUM(X))
#define DatumGetHeapTupleHeader(X) ((HeapTupleHeader) PG_DETOAST_DATUM(X))
+ /* DatumGetFoo macros for varlena2 types will typically look like this: */
+ #define DatumGetRawVarlena2P(X) ((struct varlena2 *) DatumGetPointer(X))
+ #define DatumGetVarlena2P(X) PG_DETOAST_DATUM2(X)
+ #define DatumGetStringP(X) ((String *) DatumGetVarlena2P(X))
+ #define DatumGetBinaryP(X) ((Binary *) DatumGetVarlena2P(X))
/* And we also offer variants that return an OK-to-write copy */
#define DatumGetByteaPCopy(X) ((bytea *) PG_DETOAST_DATUM_COPY(X))
#define DatumGetTextPCopy(X) ((text *) PG_DETOAST_DATUM_COPY(X))
*************** extern struct varlena *pg_detoast_datum_
*** 226,231 ****
--- 243,250 ----
/* GETARG macros for varlena types will typically look like this: */
#define PG_GETARG_BYTEA_P(n) DatumGetByteaP(PG_GETARG_DATUM(n))
#define PG_GETARG_TEXT_P(n) DatumGetTextP(PG_GETARG_DATUM(n))
+ #define PG_GETARG_STRING_P(n) DatumGetStringP(PG_GETARG_DATUM(n))
+ #define PG_GETARG_BINARY_P(n) DatumGetBinaryP(PG_GETARG_DATUM(n))
#define PG_GETARG_BPCHAR_P(n) DatumGetBpCharP(PG_GETARG_DATUM(n))
#define PG_GETARG_VARCHAR_P(n) DatumGetVarCharP(PG_GETARG_DATUM(n))
#define PG_GETARG_HEAPTUPLEHEADER(n) DatumGetHeapTupleHeader(PG_GETARG_DATUM(n))
*************** extern struct varlena *pg_detoast_datum_
*** 267,272 ****
--- 286,293 ----
/* RETURN macros for other pass-by-ref types will typically look like this: */
#define PG_RETURN_BYTEA_P(x) PG_RETURN_POINTER(x)
#define PG_RETURN_TEXT_P(x) PG_RETURN_POINTER(x)
+ #define PG_RETURN_STRING_P(x) PG_RETURN_POINTER(x)
+ #define PG_RETURN_BINARY_P(x) PG_RETURN_POINTER(x)
#define PG_RETURN_BPCHAR_P(x) PG_RETURN_POINTER(x)
#define PG_RETURN_VARCHAR_P(x) PG_RETURN_POINTER(x)
#define PG_RETURN_HEAPTUPLEHEADER(x) PG_RETURN_POINTER(x)
diff -cpr pgsql-orig/src/include/utils/builtins.h pgsql/src/include/utils/builtins.h
*** pgsql-orig/src/include/utils/builtins.h 2005-09-08 11:10:16.392311488 +0900
--- pgsql/src/include/utils/builtins.h 2005-09-08 11:11:04.906936144 +0900
*************** extern Datum to_hex64(PG_FUNCTION_ARGS);
*** 601,606 ****
--- 601,616 ----
extern Datum md5_text(PG_FUNCTION_ARGS);
extern Datum md5_bytea(PG_FUNCTION_ARGS);
+ extern Datum stringin(PG_FUNCTION_ARGS);
+ extern Datum stringout(PG_FUNCTION_ARGS);
+ extern Datum stringrecv(PG_FUNCTION_ARGS);
+ extern Datum stringsend(PG_FUNCTION_ARGS);
+
+ extern Datum binaryin(PG_FUNCTION_ARGS);
+ extern Datum binaryout(PG_FUNCTION_ARGS);
+ extern Datum binaryrecv(PG_FUNCTION_ARGS);
+ extern Datum binarysend(PG_FUNCTION_ARGS);
+
extern Datum unknownin(PG_FUNCTION_ARGS);
extern Datum unknownout(PG_FUNCTION_ARGS);
extern Datum unknownrecv(PG_FUNCTION_ARGS);
Takahiro,
PostgreSQL can treat variable-length data flexibly, but therefore
it consumes more spaces if we store short data. Headers of
variable-length types use 4 bytes regardless of the data length.My idea is to change the header itself to variable-length.
In order to reduce the size of short data, I wrote a patch to encode
lengths into the first several bits of structure. Also, the alignments
of the types were changed to 'char' from 'int'.I know my patch is still insufficient, for example, the types cannot
be TOASTed. But I guess this compression works well for short text.
Hmmm. Seems like these would be different data types from the standard ones
we deal with. I can see the value for data warehousing, for example.
Wouldn't this require creating, for example, a SHORTTEXT type? Or were you
planning this to handle VARCHAR(6) and the like? If so, how would we deal
with users who change the length via ALTER TABLE?
--
Josh Berkus
Aglio Database Solutions
San Francisco
On Thu, 2005-09-08 at 09:53 -0700, Josh Berkus wrote:
Takahiro,
PostgreSQL can treat variable-length data flexibly, but therefore
it consumes more spaces if we store short data. Headers of
variable-length types use 4 bytes regardless of the data length.My idea is to change the header itself to variable-length.
In order to reduce the size of short data, I wrote a patch to encode
lengths into the first several bits of structure. Also, the alignments
of the types were changed to 'char' from 'int'.I know my patch is still insufficient, for example, the types cannot
be TOASTed. But I guess this compression works well for short text.Hmmm. Seems like these would be different data types from the standard ones
we deal with. I can see the value for data warehousing, for example.Wouldn't this require creating, for example, a SHORTTEXT type? Or were you
planning this to handle VARCHAR(6) and the like? If so, how would we deal
with users who change the length via ALTER TABLE?
I believe ALTER TABLE always rewrites the structure using a standard
cast or another expression.
--
ITAGAKI Takahiro wrote:
# select * from pgstattuple('txttbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 57 <-- 28 + (5+3) + (5+3) + (5+3) + (5)
...# select * from pgstattuple('strtbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 36 <-- 28 + 2 + 2 + 2 + 2
What's the break even point in datum length between the two styles of
header?
And what is the processing overhead of using variable header length?
cheers
andrew
On 9/8/05 9:53 AM, "Josh Berkus" <josh@agliodbs.com> wrote:
Hmmm. Seems like these would be different data types from the standard ones
we deal with. I can see the value for data warehousing, for example.Wouldn't this require creating, for example, a SHORTTEXT type? Or were you
planning this to handle VARCHAR(6) and the like? If so, how would we deal
with users who change the length via ALTER TABLE?
If I am reading the patch correctly (and I may not be), changing the length
via ALTER TABLE would not affect anything because the header length is
variable per type instance in the table. This appears to be an alternate
encoding for VARCHAR(n) and similar.
This is not dissimilar to how ASN.1 BER prefix encodes type lengths, an
encoding widely used in wire protocols. This can save quite a few bytes in
many cases, particularly when storing short byte strings (for ASN.1, even
integers and similar are encoded this way to shave bytes on them as well but
that seems excessive). It would be useful to see how encoding/decoding cost
balances out the more compact representation in terms of performance.
J. Andrew Rogers
Josh Berkus <josh@agliodbs.com> wrote:
Wouldn't this require creating, for example, a SHORTTEXT type?
Yes, new types are required. There are no binary compatibility between
them and existing variable length types (text, bytea, etc.).
But 'SHORTTEXT' is not a proper name for them. They can represent
long texts though they are optimized for short ones.
We might be able to optimize types further if we create different types
for each length, for example, tinytext for length < 256,
shorttext for 64K, mediumtext for 16MB ...
But I think this is not appropriate. It forces users to choose one
from several text types and we will have to maintain them.
Or were you planning this to handle VARCHAR(6) and the like?
If the new text type wins VARCHAR in many respects,
I'd like to propose to replace VARCHAR with it.
---
ITAGAKI Takahiro
NTT Cyber Space Laboratories
ITAGAKI Takahiro <itagaki.takahiro@lab.ntt.co.jp> writes:
If the new text type wins VARCHAR in many respects,
I'd like to propose to replace VARCHAR with it.
This idea would impose fairly significant overhead in a number of
places, for instance locating field starts within a tuple. It didn't
appear to me that you'd even tried to measure that overhead is?
Given that slot_getattr and friends are often hotspots, I don't think
the costs involved can be ignored.
regards, tom lane
On Thu, 08 Sep 2005 18:02:44 +0900, ITAGAKI Takahiro
<itagaki.takahiro@lab.ntt.co.jp> wrote:
+ * The length of varlena2 is encoded as follows:
+ *
+ * | First | Trailing | Total | Max |
+ * | byte | bytes | bits | length |
+ * +----------+----------+-------+---------+
+ * | 0******* | 0 | 7 | 127 |
+ * | 10****** | 1 | 14 | 16K -1 |
+ * | 110***** | 2 | 21 | 2M -1 |
+ * | 1110**** | 3 | 28 | 256M -1 |
+ * | 1111---- | 4 | 32 | 4G -1 |With external and compression flags this would look like
* | ec0***** | 0 | 5 | 31 |
* | ec10**** | 1 | 12 | 4K -1 |
* | ec110*** | 2 | 19 | 512K -1 |
* | ec1110** | 3 | 26 | 64M -1 |
* | ec1111-- | 4 | 32 | 4G -1 |
Only a matter of taste, but I'd just waste one byte for sizes between 4K
and 512K and thus get a shorter table:
* | ec0***** | 0 | 5 | 31 |
* | ec10**** | 1 | 12 | 4K -1 |
* | ec110*** | 3 | 27 | 128M -1 |
* | ec111--- | 4 | 32 | 4G -1 |
And you get back that one byte for sizes between 64M and 128M :-)
Another possible tweak:
* | 0******* | 0 | 5 | 127 |
* | 1ec0**** | 1 | 12 | 4K -1 |
* | 1ec10*** | 3 | 27 | 128M -1 |
* | 1ec11--- | 4 | 32 | 4G -1 |
I.e. drop the flags for very short strings. If these flags are needed
for a string of size 32 to 127, then use a two byte header.
Servus
Manfred
Manfred Koizar <mkoi-pg@aon.at> writes:
I.e. drop the flags for very short strings.
I don't think the interaction of this idea with TOAST has been thought
through very carefully. Some points to consider:
* If the 'e' (external) bit is set, the actual length is 20 or so bytes.
Always. There is no need for an 'e' bit in the variants that store a
longer length.
* I agree with Manfred that you could blow off the case of compressing
strings shorter than 128 bytes, so you really only need the 'c' bit
for the longer variants. Hence, just one flag bit is needed in all
cases.
* The proposal to reduce the alignment to char breaks TOAST pointers,
which contain fields that need to be word-aligned (at least on machines
that are picky about alignment). Maybe we could just live with that,
by copying the in-datum data to and from properly aligned local storage
in the routines that need to access the pointer fields. It's an extra
cost though, as well as a probable source of bugs that will only be
noticed on non-Intel hardware.
* In the 'c' case you need to store *two* lengths, the physical length
and the decompressed length. Is it worth playing the same kind of
game with the decompressed length? Again, you can't just do nothing
because the proposal breaks word-alignment of the decompressed length.
Another thing that's bothering me about this is that it'll lead to weird
new coding rules that will certainly break a lot of code; notably "you
can't access VARDATA(x) until you've assigned the correct value to
VARSIZE(x)." This will be a particular pain in the neck for code that
likes to fill in the data before it makes a final determination of the
result size.
So I'm feeling a good deal of sales resistance to the idea that this
ought to replace our existing varlena representation. At least for
internal use. Maybe the right way to think about this is that a
compressed length word is a form of TOASTing, which we apply during
toast compression and undo during PG_DETOAST_DATUM. If we did that,
the changes would be vastly more localized than if we try to make
all the datatype manipulation routines cope. Not sure about the
speed tradeoffs involved, though.
regards, tom lane
I assume the conclusion from this email thread is that though the idea
is interesting, the complexity added would not be worth the saving of a
few bytes.
---------------------------------------------------------------------------
ITAGAKI Takahiro wrote:
Hi Hackers,
PostgreSQL can treat variable-length data flexibly, but therefore
it consumes more spaces if we store short data. Headers of
variable-length types use 4 bytes regardless of the data length.My idea is to change the header itself to variable-length.
In order to reduce the size of short data, I wrote a patch to encode
lengths into the first several bits of structure. Also, the alignments
of the types were changed to 'char' from 'int'.I know my patch is still insufficient, for example, the types cannot
be TOASTed. But I guess this compression works well for short text.I'll appreciate any comments.
thanks.---- the result of patch ----
# create table txttbl (v1 text, v2 text, v3 text, v4 text);
# create table strtbl (v1 string, v2 string, v3 string, v4 string);# insert into txttbl values('A', 'B', 'C', 'D');
# insert into strtbl values('A', 'B', 'C', 'D');# select * from pgstattuple('txttbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 57 <-- 28 + (5+3) + (5+3) + (5+3) + (5)
...# select * from pgstattuple('strtbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 36 <-- 28 + 2 + 2 + 2 + 2
...---
ITAGAKI Takahiro
NTT Cyber Space Laboratories
[ Attachment, skipping... ]
---------------------------(end of broadcast)---------------------------
TIP 6: explain analyze is your friend
--
Bruce Momjian http://candle.pha.pa.us
EnterpriseDB http://www.enterprisedb.com
+ If your life is a hard drive, Christ can be your backup. +
On Wed, Jun 14, 2006 at 02:53:10PM -0400, Bruce Momjian wrote:
I assume the conclusion from this email thread is that though the idea
is interesting, the complexity added would not be worth the saving of a
few bytes.
Anyone do any testing?
I'm also wondering if this would be useful to allow fields larger than
1G.
---------------------------------------------------------------------------
ITAGAKI Takahiro wrote:
Hi Hackers,
PostgreSQL can treat variable-length data flexibly, but therefore
it consumes more spaces if we store short data. Headers of
variable-length types use 4 bytes regardless of the data length.My idea is to change the header itself to variable-length.
In order to reduce the size of short data, I wrote a patch to encode
lengths into the first several bits of structure. Also, the alignments
of the types were changed to 'char' from 'int'.I know my patch is still insufficient, for example, the types cannot
be TOASTed. But I guess this compression works well for short text.I'll appreciate any comments.
thanks.---- the result of patch ----
# create table txttbl (v1 text, v2 text, v3 text, v4 text);
# create table strtbl (v1 string, v2 string, v3 string, v4 string);# insert into txttbl values('A', 'B', 'C', 'D');
# insert into strtbl values('A', 'B', 'C', 'D');# select * from pgstattuple('txttbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 57 <-- 28 + (5+3) + (5+3) + (5+3) + (5)
...# select * from pgstattuple('strtbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 36 <-- 28 + 2 + 2 + 2 + 2
...---
ITAGAKI Takahiro
NTT Cyber Space Laboratories[ Attachment, skipping... ]
---------------------------(end of broadcast)---------------------------
TIP 6: explain analyze is your friend--
Bruce Momjian http://candle.pha.pa.us
EnterpriseDB http://www.enterprisedb.com+ If your life is a hard drive, Christ can be your backup. +
---------------------------(end of broadcast)---------------------------
TIP 4: Have you searched our list archives?
--
Jim C. Nasby, Sr. Engineering Consultant jnasby@pervasive.com
Pervasive Software http://pervasive.com work: 512-231-6117
vcard: http://jim.nasby.net/pervasive.vcf cell: 512-569-9461
Jim C. Nasby wrote:
On Wed, Jun 14, 2006 at 02:53:10PM -0400, Bruce Momjian wrote:
I assume the conclusion from this email thread is that though the idea
is interesting, the complexity added would not be worth the saving of a
few bytes.Anyone do any testing?
I'm also wondering if this would be useful to allow fields larger than
1G.
The submitter showed the pathological case where a single char was
stored in a text field, and showed the reduced size (below). There were
no performance numbers given. It seems like an edge case, especially
since we have a "char" type that is a single byte.
---------------------------------------------------------------------------
ITAGAKI Takahiro wrote:
Hi Hackers,
PostgreSQL can treat variable-length data flexibly, but therefore
it consumes more spaces if we store short data. Headers of
variable-length types use 4 bytes regardless of the data length.My idea is to change the header itself to variable-length.
In order to reduce the size of short data, I wrote a patch to encode
lengths into the first several bits of structure. Also, the alignments
of the types were changed to 'char' from 'int'.I know my patch is still insufficient, for example, the types cannot
be TOASTed. But I guess this compression works well for short text.I'll appreciate any comments.
thanks.---- the result of patch ----
# create table txttbl (v1 text, v2 text, v3 text, v4 text);
# create table strtbl (v1 string, v2 string, v3 string, v4 string);# insert into txttbl values('A', 'B', 'C', 'D');
# insert into strtbl values('A', 'B', 'C', 'D');# select * from pgstattuple('txttbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 57 <-- 28 + (5+3) + (5+3) + (5+3) + (5)
...# select * from pgstattuple('strtbl');
-[ RECORD 1 ]------+------
table_len | 8192
tuple_count | 1
tuple_len | 36 <-- 28 + 2 + 2 + 2 + 2
...---
ITAGAKI Takahiro
NTT Cyber Space Laboratories[ Attachment, skipping... ]
---------------------------(end of broadcast)---------------------------
TIP 6: explain analyze is your friend--
Bruce Momjian http://candle.pha.pa.us
EnterpriseDB http://www.enterprisedb.com+ If your life is a hard drive, Christ can be your backup. +
---------------------------(end of broadcast)---------------------------
TIP 4: Have you searched our list archives?--
Jim C. Nasby, Sr. Engineering Consultant jnasby@pervasive.com
Pervasive Software http://pervasive.com work: 512-231-6117
vcard: http://jim.nasby.net/pervasive.vcf cell: 512-569-9461---------------------------(end of broadcast)---------------------------
TIP 1: if posting/reading through Usenet, please send an appropriate
subscribe-nomail command to majordomo@postgresql.org so that your
message can get through to the mailing list cleanly
--
Bruce Momjian http://candle.pha.pa.us
EnterpriseDB http://www.enterprisedb.com
+ If your life is a hard drive, Christ can be your backup. +
On Wed, Jun 14, 2006 at 04:21:34PM -0400, Bruce Momjian wrote:
Jim C. Nasby wrote:
On Wed, Jun 14, 2006 at 02:53:10PM -0400, Bruce Momjian wrote:
I assume the conclusion from this email thread is that though the idea
is interesting, the complexity added would not be worth the saving of a
few bytes.Anyone do any testing?
I'm also wondering if this would be useful to allow fields larger than
1G.The submitter showed the pathological case where a single char was
stored in a text field, and showed the reduced size (below). There were
no performance numbers given. It seems like an edge case, especially
since we have a "char" type that is a single byte.
Well, depending on how the patch works I could see it being valuable for
tables that have a number of 'short' text fields, where short is less
than 127 bytes.
I've got some tables like that I can test on, at least to see the size
difference. Not really sure what a valid performance test would be,
though...
I'm wondering if it would be worth trying to organize users to do
testing of stuff like this. I'm sure there's lots of folks who know how
to apply a patch and have test data that could benefit from patches like
this. (I'm assuming this patch didn't place any substantial performance
penalties into the backend...)
--
Jim C. Nasby, Sr. Engineering Consultant jnasby@pervasive.com
Pervasive Software http://pervasive.com work: 512-231-6117
vcard: http://jim.nasby.net/pervasive.vcf cell: 512-569-9461
The code churn to do this is going to be quite significant, as well a
performance-wise hit perhaps, so it has to be a big win.
---------------------------------------------------------------------------
Jim C. Nasby wrote:
On Wed, Jun 14, 2006 at 04:21:34PM -0400, Bruce Momjian wrote:
Jim C. Nasby wrote:
On Wed, Jun 14, 2006 at 02:53:10PM -0400, Bruce Momjian wrote:
I assume the conclusion from this email thread is that though the idea
is interesting, the complexity added would not be worth the saving of a
few bytes.Anyone do any testing?
I'm also wondering if this would be useful to allow fields larger than
1G.The submitter showed the pathological case where a single char was
stored in a text field, and showed the reduced size (below). There were
no performance numbers given. It seems like an edge case, especially
since we have a "char" type that is a single byte.Well, depending on how the patch works I could see it being valuable for
tables that have a number of 'short' text fields, where short is less
than 127 bytes.I've got some tables like that I can test on, at least to see the size
difference. Not really sure what a valid performance test would be,
though...I'm wondering if it would be worth trying to organize users to do
testing of stuff like this. I'm sure there's lots of folks who know how
to apply a patch and have test data that could benefit from patches like
this. (I'm assuming this patch didn't place any substantial performance
penalties into the backend...)
--
Jim C. Nasby, Sr. Engineering Consultant jnasby@pervasive.com
Pervasive Software http://pervasive.com work: 512-231-6117
vcard: http://jim.nasby.net/pervasive.vcf cell: 512-569-9461
--
Bruce Momjian http://candle.pha.pa.us
EnterpriseDB http://www.enterprisedb.com
+ If your life is a hard drive, Christ can be your backup. +
On Wed, 2006-06-14 at 16:56 -0400, Bruce Momjian wrote:
The code churn to do this is going to be quite significant
Why do you say that? Although the original patch posted to implement
this was incomplete, it certainly wasn't very large.
-Neil
Bruce Momjian <pgman@candle.pha.pa.us> wrote:
I assume the conclusion from this email thread is that though the idea
is interesting, the complexity added would not be worth the saving of a
few bytes.
I have same understanding about it. Besides the complexity,
there is trade-off between cpu and i/o resources.
Also, there is another approach in
http://archives.postgresql.org/pgsql-hackers/2005-12/msg00258.php ,
where 2byte header version of varlena is suggested. It is more simple, but
it may double intrinsic text types (shorttext, shortchar and shortvarchar).
I assume it is worth the saving of a few bytes in particular for indexes
on VLDBs, but my patch is still incomplete and needs more works.
---
ITAGAKI Takahiro
NTT Open Source Software Center
Neil Conway <neilc@samurai.com> writes:
On Wed, 2006-06-14 at 16:56 -0400, Bruce Momjian wrote:
The code churn to do this is going to be quite significant
Why do you say that? Although the original patch posted to implement
this was incomplete, it certainly wasn't very large.
IIRC, the original idea would have forced code changes on every existing
varlena datatype ... not just those in our source tree, but outside
users' custom types as well. You'd need a really *serious* gain to
justify that.
The second idea was to leave existing varlena types as-is and invent
a set of "small text" etc datatypes alongside them. This is simply
ugly (shades of Oracle's varchar2), and again there wasn't really an
adequate case made why we should burden users with extra complexity.
The variant I could have supported was making a "2-byte-header" class of
types and using it just for the few types where it would be sufficient
(numeric and inet, basically). That would be transparent to everyone.
Whether it's worth the trouble is not clear given that it'd only help a
few datatypes, but would add cycles in critical inner loops in places
like heap_deformtuple.
regards, tom lane