From 413cd728ba7db42f23315b08ee00c163d3f1c7eb Mon Sep 17 00:00:00 2001 From: Nikita Glukhov Date: Fri, 9 Feb 2018 18:34:29 +0300 Subject: [PATCH 8/9] Add opclass parameters to contrib/hstore --- contrib/hstore/Makefile | 2 +- contrib/hstore/hstore--1.5--1.6.sql | 32 ++++++ contrib/hstore/hstore.control | 2 +- contrib/hstore/hstore_gist.c | 200 ++++++++++++++++++++---------------- doc/src/sgml/hstore.sgml | 9 ++ 5 files changed, 154 insertions(+), 91 deletions(-) create mode 100644 contrib/hstore/hstore--1.5--1.6.sql diff --git a/contrib/hstore/Makefile b/contrib/hstore/Makefile index 46d26f8..a8a5105 100644 --- a/contrib/hstore/Makefile +++ b/contrib/hstore/Makefile @@ -5,7 +5,7 @@ OBJS = hstore_io.o hstore_op.o hstore_gist.o hstore_gin.o hstore_compat.o \ $(WIN32RES) EXTENSION = hstore -DATA = hstore--1.4.sql hstore--1.4--1.5.sql \ +DATA = hstore--1.4.sql hstore--1.4--1.5.sql hstore--1.5--1.6.sql \ hstore--1.3--1.4.sql hstore--1.2--1.3.sql \ hstore--1.1--1.2.sql hstore--1.0--1.1.sql \ hstore--unpackaged--1.0.sql diff --git a/contrib/hstore/hstore--1.5--1.6.sql b/contrib/hstore/hstore--1.5--1.6.sql new file mode 100644 index 0000000..08c85ee --- /dev/null +++ b/contrib/hstore/hstore--1.5--1.6.sql @@ -0,0 +1,32 @@ +/* contrib/hstore/hstore--1.5--1.6.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION intarray UPDATE TO '1.6'" to load this file. \quit + +-- Update procedure signatures the hard way. +-- We use to_regprocedure() so that query doesn't fail if run against 9.6beta1 definitions, +-- wherein the signatures have been updated already. In that case to_regprocedure() will +-- return NULL and no updates will happen. + +UPDATE pg_catalog.pg_proc SET + proargtypes = pg_catalog.array_to_string(newtypes::pg_catalog.oid[], ' ')::pg_catalog.oidvector, + pronargs = pg_catalog.array_length(newtypes, 1) +FROM (VALUES +(NULL::pg_catalog.text, NULL::pg_catalog.regtype[]), -- establish column types +('ghstore_compress(internal)', '{internal,internal}'), +('ghstore_decompress(internal)', '{internal,internal}'), +('ghstore_consistent(internal,hstore,smallint,oid,internal)', '{internal,hstore,smallint,oid,internal,internal}'), +('ghstore_penalty(internal,internal,internal)', '{internal,internal,internal,internal}'), +('ghstore_picksplit(internal,internal)', '{internal,internal,internal}'), +('ghstore_union(internal,internal)', '{internal,internal,internal}'), +('ghstore_same(ghstore,ghstore,internal)', '{ghstore,ghstore,internal,internal}') +) AS update_data (oldproc, newtypes) +WHERE oid = pg_catalog.to_regprocedure(oldproc); + +CREATE FUNCTION ghstore_options(internal, boolean) +RETURNS internal +AS 'MODULE_PATHNAME', 'ghstore_options' +LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +ALTER OPERATOR FAMILY gist_hstore_ops USING gist +ADD FUNCTION 10 (hstore) ghstore_options (internal, boolean); diff --git a/contrib/hstore/hstore.control b/contrib/hstore/hstore.control index 8a71947..93688cd 100644 --- a/contrib/hstore/hstore.control +++ b/contrib/hstore/hstore.control @@ -1,5 +1,5 @@ # hstore extension comment = 'data type for storing sets of (key, value) pairs' -default_version = '1.5' +default_version = '1.6' module_pathname = '$libdir/hstore' relocatable = true diff --git a/contrib/hstore/hstore_gist.c b/contrib/hstore/hstore_gist.c index 6d24d2f..13be2c0 100644 --- a/contrib/hstore/hstore_gist.c +++ b/contrib/hstore/hstore_gist.c @@ -4,29 +4,33 @@ #include "postgres.h" #include "access/gist.h" +#include "access/reloptions.h" #include "access/stratnum.h" #include "catalog/pg_type.h" #include "utils/pg_crc.h" #include "hstore.h" +/* gist_hstore_ops opclass options */ +typedef struct HstoreGistOptions +{ + int32 vl_len_; /* varlena header (do not touch directly!) */ + int siglen; /* signature length in bytes */ +} GistHstoreOptions; + /* bigint defines */ #define BITBYTE 8 -#define SIGLENINT 4 /* >122 => key will toast, so very slow!!! */ -#define SIGLEN ( sizeof(int)*SIGLENINT ) -#define SIGLENBIT (SIGLEN*BITBYTE) +#define SIGLEN_DEFAULT (sizeof(int32) * 4) +#define SIGLEN_MAX (sizeof(int32) * 122) /* key will toast, so very slow!!! */ +#define SIGLENBIT(siglen) ((siglen) * BITBYTE) -typedef char BITVEC[SIGLEN]; typedef char *BITVECP; -#define SIGPTR(x) ( (BITVECP) ARR_DATA_PTR(x) ) - +#define LOOPBYTE(siglen) \ + for (i = 0; i < (siglen); i++) -#define LOOPBYTE \ - for(i=0;i> ( (i) % BITBYTE )) & 0x01 ) -#define HASHVAL(val) (((unsigned int)(val)) % SIGLENBIT) -#define HASH(sign, val) SETBIT((sign), HASHVAL(val)) +#define HASHVAL(val, siglen) (((unsigned int)(val)) % SIGLENBIT(siglen)) +#define HASH(sign, val, siglen) SETBIT((sign), HASHVAL(val, siglen)) typedef struct { @@ -49,7 +53,7 @@ typedef struct #define ISALLTRUE(x) ( ((GISTTYPE*)x)->flag & ALLISTRUE ) #define GTHDRSIZE (VARHDRSZ + sizeof(int32)) -#define CALCGTSIZE(flag) ( GTHDRSIZE+(((flag) & ALLISTRUE) ? 0 : SIGLEN) ) +#define CALCGTSIZE(flag, siglen) ( GTHDRSIZE+(((flag) & ALLISTRUE) ? 0 : (siglen)) ) #define GETSIGN(x) ( (BITVECP)( (char*)x+GTHDRSIZE ) ) @@ -100,6 +104,27 @@ ghstore_out(PG_FUNCTION_ARGS) PG_RETURN_DATUM(0); } +static GISTTYPE * +ghstore_alloc(bool allistrue, int siglen, BITVECP sign) +{ + int flag = allistrue ? ALLISTRUE : 0; + int size = CALCGTSIZE(flag, siglen); + GISTTYPE *res = palloc(size); + + SET_VARSIZE(res, size); + res->flag = flag; + + if (!allistrue) + { + if (sign) + memcpy(GETSIGN(res), sign, siglen); + else + memset(GETSIGN(res), 0, siglen); + } + + return res; +} + PG_FUNCTION_INFO_V1(ghstore_consistent); PG_FUNCTION_INFO_V1(ghstore_compress); PG_FUNCTION_INFO_V1(ghstore_decompress); @@ -107,36 +132,37 @@ PG_FUNCTION_INFO_V1(ghstore_penalty); PG_FUNCTION_INFO_V1(ghstore_picksplit); PG_FUNCTION_INFO_V1(ghstore_union); PG_FUNCTION_INFO_V1(ghstore_same); +PG_FUNCTION_INFO_V1(ghstore_options); Datum ghstore_compress(PG_FUNCTION_ARGS) { GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + GistHstoreOptions *options = (GistHstoreOptions *) PG_GETARG_POINTER(1); + int siglen = options->siglen; GISTENTRY *retval = entry; if (entry->leafkey) { - GISTTYPE *res = (GISTTYPE *) palloc0(CALCGTSIZE(0)); + GISTTYPE *res = ghstore_alloc(false, siglen, NULL); HStore *val = DatumGetHStoreP(entry->key); HEntry *hsent = ARRPTR(val); char *ptr = STRPTR(val); int count = HS_COUNT(val); int i; - SET_VARSIZE(res, CALCGTSIZE(0)); - for (i = 0; i < count; ++i) { int h; h = crc32_sz((char *) HSTORE_KEY(hsent, ptr, i), HSTORE_KEYLEN(hsent, i)); - HASH(GETSIGN(res), h); + HASH(GETSIGN(res), h, siglen); if (!HSTORE_VALISNULL(hsent, i)) { h = crc32_sz((char *) HSTORE_VAL(hsent, ptr, i), HSTORE_VALLEN(hsent, i)); - HASH(GETSIGN(res), h); + HASH(GETSIGN(res), h, siglen); } } @@ -152,15 +178,13 @@ ghstore_compress(PG_FUNCTION_ARGS) GISTTYPE *res; BITVECP sign = GETSIGN(DatumGetPointer(entry->key)); - LOOPBYTE + LOOPBYTE(siglen) { if ((sign[i] & 0xff) != 0xff) PG_RETURN_POINTER(retval); } - res = (GISTTYPE *) palloc(CALCGTSIZE(ALLISTRUE)); - SET_VARSIZE(res, CALCGTSIZE(ALLISTRUE)); - res->flag = ALLISTRUE; + res = ghstore_alloc(true, siglen, NULL); retval = (GISTENTRY *) palloc(sizeof(GISTENTRY)); gistentryinit(*retval, PointerGetDatum(res), @@ -188,6 +212,9 @@ ghstore_same(PG_FUNCTION_ARGS) GISTTYPE *a = (GISTTYPE *) PG_GETARG_POINTER(0); GISTTYPE *b = (GISTTYPE *) PG_GETARG_POINTER(1); bool *result = (bool *) PG_GETARG_POINTER(2); + GistHstoreOptions *options = (GistHstoreOptions *) PG_GETARG_POINTER(3); + int siglen = options->siglen; + if (ISALLTRUE(a) && ISALLTRUE(b)) *result = true; @@ -202,7 +229,7 @@ ghstore_same(PG_FUNCTION_ARGS) sb = GETSIGN(b); *result = true; - LOOPBYTE + LOOPBYTE(siglen) { if (sa[i] != sb[i]) { @@ -215,12 +242,12 @@ ghstore_same(PG_FUNCTION_ARGS) } static int32 -sizebitvec(BITVECP sign) +sizebitvec(BITVECP sign, int siglen) { int32 size = 0, i; - LOOPBYTE + LOOPBYTE(siglen) { size += SUMBIT(sign); sign = (BITVECP) (((char *) sign) + 1); @@ -229,12 +256,12 @@ sizebitvec(BITVECP sign) } static int -hemdistsign(BITVECP a, BITVECP b) +hemdistsign(BITVECP a, BITVECP b, int siglen) { int i, dist = 0; - LOOPBIT + LOOPBIT(siglen) { if (GETBIT(a, i) != GETBIT(b, i)) dist++; @@ -243,30 +270,30 @@ hemdistsign(BITVECP a, BITVECP b) } static int -hemdist(GISTTYPE *a, GISTTYPE *b) +hemdist(GISTTYPE *a, GISTTYPE *b, int siglen) { if (ISALLTRUE(a)) { if (ISALLTRUE(b)) return 0; else - return SIGLENBIT - sizebitvec(GETSIGN(b)); + return SIGLENBIT(siglen) - sizebitvec(GETSIGN(b), siglen); } else if (ISALLTRUE(b)) - return SIGLENBIT - sizebitvec(GETSIGN(a)); + return SIGLENBIT(siglen) - sizebitvec(GETSIGN(a), siglen); - return hemdistsign(GETSIGN(a), GETSIGN(b)); + return hemdistsign(GETSIGN(a), GETSIGN(b), siglen); } static int32 -unionkey(BITVECP sbase, GISTTYPE *add) +unionkey(BITVECP sbase, GISTTYPE *add, int siglen) { int32 i; BITVECP sadd = GETSIGN(add); if (ISALLTRUE(add)) return 1; - LOOPBYTE + LOOPBYTE(siglen) sbase[i] |= sadd[i]; return 0; } @@ -278,28 +305,23 @@ ghstore_union(PG_FUNCTION_ARGS) int32 len = entryvec->n; int *size = (int *) PG_GETARG_POINTER(1); - BITVEC base; + GistHstoreOptions *options = (GistHstoreOptions *) PG_GETARG_POINTER(2); + int siglen = options->siglen; int32 i; - int32 flag = 0; - GISTTYPE *result; + GISTTYPE *result = ghstore_alloc(false, siglen, NULL); + BITVECP base = GETSIGN(result); - MemSet((void *) base, 0, sizeof(BITVEC)); for (i = 0; i < len; i++) { - if (unionkey(base, GETENTRY(entryvec, i))) + if (unionkey(base, GETENTRY(entryvec, i), siglen)) { - flag = ALLISTRUE; + result->flag |= ALLISTRUE; + SET_VARSIZE(result, CALCGTSIZE(ALLISTRUE, siglen)); break; } } - len = CALCGTSIZE(flag); - result = (GISTTYPE *) palloc(len); - SET_VARSIZE(result, len); - result->flag = flag; - if (!ISALLTRUE(result)) - memcpy((void *) GETSIGN(result), (void *) base, sizeof(BITVEC)); - *size = len; + *size = VARSIZE(result); PG_RETURN_POINTER(result); } @@ -310,10 +332,12 @@ ghstore_penalty(PG_FUNCTION_ARGS) GISTENTRY *origentry = (GISTENTRY *) PG_GETARG_POINTER(0); /* always ISSIGNKEY */ GISTENTRY *newentry = (GISTENTRY *) PG_GETARG_POINTER(1); float *penalty = (float *) PG_GETARG_POINTER(2); + GistHstoreOptions *options = (GistHstoreOptions *) PG_GETARG_POINTER(3); + int siglen = options->siglen; GISTTYPE *origval = (GISTTYPE *) DatumGetPointer(origentry->key); GISTTYPE *newval = (GISTTYPE *) DatumGetPointer(newentry->key); - *penalty = hemdist(origval, newval); + *penalty = hemdist(origval, newval, siglen); PG_RETURN_POINTER(penalty); } @@ -338,6 +362,8 @@ ghstore_picksplit(PG_FUNCTION_ARGS) OffsetNumber maxoff = entryvec->n - 2; GIST_SPLITVEC *v = (GIST_SPLITVEC *) PG_GETARG_POINTER(1); + GistHstoreOptions *options = (GistHstoreOptions *) PG_GETARG_POINTER(2); + int siglen = options->siglen; OffsetNumber k, j; GISTTYPE *datum_l, @@ -368,7 +394,7 @@ ghstore_picksplit(PG_FUNCTION_ARGS) _k = GETENTRY(entryvec, k); for (j = OffsetNumberNext(k); j <= maxoff; j = OffsetNumberNext(j)) { - size_waste = hemdist(_k, GETENTRY(entryvec, j)); + size_waste = hemdist(_k, GETENTRY(entryvec, j), siglen); if (size_waste > waste) { waste = size_waste; @@ -390,33 +416,10 @@ ghstore_picksplit(PG_FUNCTION_ARGS) } /* form initial .. */ - if (ISALLTRUE(GETENTRY(entryvec, seed_1))) - { - datum_l = (GISTTYPE *) palloc(GTHDRSIZE); - SET_VARSIZE(datum_l, GTHDRSIZE); - datum_l->flag = ALLISTRUE; - } - else - { - datum_l = (GISTTYPE *) palloc(GTHDRSIZE + SIGLEN); - SET_VARSIZE(datum_l, GTHDRSIZE + SIGLEN); - datum_l->flag = 0; - memcpy((void *) GETSIGN(datum_l), (void *) GETSIGN(GETENTRY(entryvec, seed_1)), sizeof(BITVEC)) - ; - } - if (ISALLTRUE(GETENTRY(entryvec, seed_2))) - { - datum_r = (GISTTYPE *) palloc(GTHDRSIZE); - SET_VARSIZE(datum_r, GTHDRSIZE); - datum_r->flag = ALLISTRUE; - } - else - { - datum_r = (GISTTYPE *) palloc(GTHDRSIZE + SIGLEN); - SET_VARSIZE(datum_r, GTHDRSIZE + SIGLEN); - datum_r->flag = 0; - memcpy((void *) GETSIGN(datum_r), (void *) GETSIGN(GETENTRY(entryvec, seed_2)), sizeof(BITVEC)); - } + datum_l = ghstore_alloc(ISALLTRUE(GETENTRY(entryvec, seed_1)), siglen, + GETSIGN(GETENTRY(entryvec, seed_1))); + datum_r = ghstore_alloc(ISALLTRUE(GETENTRY(entryvec, seed_2)), siglen, + GETSIGN(GETENTRY(entryvec, seed_2))); maxoff = OffsetNumberNext(maxoff); /* sort before ... */ @@ -425,8 +428,8 @@ ghstore_picksplit(PG_FUNCTION_ARGS) { costvector[j - 1].pos = j; _j = GETENTRY(entryvec, j); - size_alpha = hemdist(datum_l, _j); - size_beta = hemdist(datum_r, _j); + size_alpha = hemdist(datum_l, _j, siglen); + size_beta = hemdist(datum_r, _j, siglen); costvector[j - 1].cost = abs(size_alpha - size_beta); } qsort((void *) costvector, maxoff, sizeof(SPLITCOST), comparecost); @@ -450,20 +453,20 @@ ghstore_picksplit(PG_FUNCTION_ARGS) continue; } _j = GETENTRY(entryvec, j); - size_alpha = hemdist(datum_l, _j); - size_beta = hemdist(datum_r, _j); + size_alpha = hemdist(datum_l, _j, siglen); + size_beta = hemdist(datum_r, _j, siglen); if (size_alpha < size_beta + WISH_F(v->spl_nleft, v->spl_nright, 0.0001)) { if (ISALLTRUE(datum_l) || ISALLTRUE(_j)) { if (!ISALLTRUE(datum_l)) - MemSet((void *) union_l, 0xff, sizeof(BITVEC)); + MemSet((void *) union_l, 0xff, siglen); } else { ptr = GETSIGN(_j); - LOOPBYTE + LOOPBYTE(siglen) union_l[i] |= ptr[i]; } *left++ = j; @@ -474,12 +477,12 @@ ghstore_picksplit(PG_FUNCTION_ARGS) if (ISALLTRUE(datum_r) || ISALLTRUE(_j)) { if (!ISALLTRUE(datum_r)) - MemSet((void *) union_r, 0xff, sizeof(BITVEC)); + MemSet((void *) union_r, 0xff, siglen); } else { ptr = GETSIGN(_j); - LOOPBYTE + LOOPBYTE(siglen) union_r[i] |= ptr[i]; } *right++ = j; @@ -504,6 +507,8 @@ ghstore_consistent(PG_FUNCTION_ARGS) /* Oid subtype = PG_GETARG_OID(3); */ bool *recheck = (bool *) PG_GETARG_POINTER(4); + GistHstoreOptions *options = (GistHstoreOptions *) PG_GETARG_POINTER(5); + int siglen = options->siglen; bool res = true; BITVECP sign; @@ -529,13 +534,13 @@ ghstore_consistent(PG_FUNCTION_ARGS) int crc = crc32_sz((char *) HSTORE_KEY(qe, qv, i), HSTORE_KEYLEN(qe, i)); - if (GETBIT(sign, HASHVAL(crc))) + if (GETBIT(sign, HASHVAL(crc, siglen))) { if (!HSTORE_VALISNULL(qe, i)) { crc = crc32_sz((char *) HSTORE_VAL(qe, qv, i), HSTORE_VALLEN(qe, i)); - if (!GETBIT(sign, HASHVAL(crc))) + if (!GETBIT(sign, HASHVAL(crc, siglen))) res = false; } } @@ -548,7 +553,7 @@ ghstore_consistent(PG_FUNCTION_ARGS) text *query = PG_GETARG_TEXT_PP(1); int crc = crc32_sz(VARDATA_ANY(query), VARSIZE_ANY_EXHDR(query)); - res = (GETBIT(sign, HASHVAL(crc))) ? true : false; + res = (GETBIT(sign, HASHVAL(crc, siglen))) ? true : false; } else if (strategy == HStoreExistsAllStrategyNumber) { @@ -569,7 +574,7 @@ ghstore_consistent(PG_FUNCTION_ARGS) if (key_nulls[i]) continue; crc = crc32_sz(VARDATA(key_datums[i]), VARSIZE(key_datums[i]) - VARHDRSZ); - if (!(GETBIT(sign, HASHVAL(crc)))) + if (!(GETBIT(sign, HASHVAL(crc, siglen)))) res = false; } } @@ -594,7 +599,7 @@ ghstore_consistent(PG_FUNCTION_ARGS) if (key_nulls[i]) continue; crc = crc32_sz(VARDATA(key_datums[i]), VARSIZE(key_datums[i]) - VARHDRSZ); - if (GETBIT(sign, HASHVAL(crc))) + if (GETBIT(sign, HASHVAL(crc, siglen))) res = true; } } @@ -603,3 +608,20 @@ ghstore_consistent(PG_FUNCTION_ARGS) PG_RETURN_BOOL(res); } + +Datum +ghstore_options(PG_FUNCTION_ARGS) +{ + Datum raw_options = PG_GETARG_DATUM(0); + bool validate = PG_GETARG_BOOL(1); + relopt_int siglen = + { {"siglen", "signature length", 0, 0, 6, RELOPT_TYPE_INT }, + SIGLEN_DEFAULT, 1, SIGLEN_MAX }; + relopt_gen *optgen[] = { &siglen.gen }; + int offsets[] = { offsetof(GistHstoreOptions, siglen) }; + GistHstoreOptions *options = + parseAndFillLocalRelOptions(raw_options, optgen, offsets, 1, + sizeof(GistHstoreOptions), validate); + + PG_RETURN_POINTER(options); +} diff --git a/doc/src/sgml/hstore.sgml b/doc/src/sgml/hstore.sgml index 94ccd12..45a811f 100644 --- a/doc/src/sgml/hstore.sgml +++ b/doc/src/sgml/hstore.sgml @@ -462,6 +462,15 @@ CREATE INDEX hidx ON testhstore USING GIN (h); + GiST opclass gist_hstore_ops has one optional parameter + siglen for specifying signature length in bytes + (default signature length is 16 bytes, maximal length is 488 bytes): + + CREATE INDEX hidx ON testhstore USING GIST (h gist_hstore_ops(siglen=32)); + + + + hstore also supports btree or hash indexes for the = operator. This allows hstore columns to be declared UNIQUE, or to be used in GROUP BY, -- 2.7.4