WIP: shared ispell dictionary
Hello
attached patch add possibility to share ispell dictionary between
processes. The reason for this is the slowness of first tsearch query
and size of allocated memory per process. When I tested loading of
ispell dictionary (for Czech language) I got about 500 ms and 48MB.
With simple allocator it uses only 25 MB. If we remove some check and
tolower string transformation from loading stage it needs only 200 ms.
But with broken dict or affix file it can put wrong results. This
patch significantly reduce load on servers that use ispell
dictionaries.
I know so Tom worries about using of share memory. I think so it
unnecessarily. After loading data from dictionary are only read, never
modified. Second idea - this dictionary template can be distributed as
separate project (it needs a few changes in core - and simple
allocator).
Using:
a) set shared_data = 26MB (postgres.conf)
b) restart
c) register dictionary with option "share=yes"
CREATE TEXT SEARCH DICTIONARY cspell
(template=ispell, dictfile = czech, afffile=czech, stopwords=czech,
share = yes);
[pavel@nemesis src]$ psql-dev3 postgres
Timing is on.
psql-dev3 (9.0devel)
Type "help" for help.
postgres=# select * from ts_debug('cs','Příliš žluťoučký kůň se napil
žluté vody');
alias | description | token | dictionaries |
dictionary | lexemes
-----------+-------------------+-----------+-----------------+------------+-------------
word | Word, all letters | Příliš | {cspell,simple} | cspell
| {příliš}
blank | Space symbols | | {} | |
word | Word, all letters | žluťoučký | {cspell,simple} | cspell
| {žluťoučký}
blank | Space symbols | | {} | |
word | Word, all letters | kůň | {cspell,simple} | cspell
| {kůň}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | se | {cspell,simple} | cspell | {}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | napil | {cspell,simple} | cspell
| {napít}
blank | Space symbols | | {} | |
word | Word, all letters | žluté | {cspell,simple} | cspell
| {žlutý}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | vody | {cspell,simple} | cspell
| {voda}
(13 rows)
Time: 8,178 ms <<-- without patch 500ms
Limits and ToDo:
a) it support only simple regular expressions
b) it doesn't solve cache reset a shared memory deallocation
Regards
Pavel Stehule
Attachments:
shared_dictionary_02.diffapplication/octet-stream; name=shared_dictionary_02.diffDownload
*** ./src/backend/storage/ipc/ipci.c.orig 2010-03-17 18:36:47.332657055 +0100
--- ./src/backend/storage/ipc/ipci.c 2010-03-18 07:48:42.559964237 +0100
***************
*** 43,48 ****
--- 43,50 ----
static Size total_addin_request = 0;
static bool addin_request_allowed = true;
+ extern int shared_data;
+
/*
* RequestAddinShmemSpace
***************
*** 132,137 ****
--- 134,142 ----
addin_request_allowed = false;
size = add_size(size, total_addin_request);
+ /* reserve shared memory for possible common data as ispell dictionaries */
+ size = add_size(size, shared_data * 1024);
+
/* might as well round it off to a multiple of a typical page size */
size = add_size(size, 8192 - (size % 8192));
*** ./src/backend/tsearch/dict_ispell.c.orig 2010-03-15 15:10:31.997321968 +0100
--- ./src/backend/tsearch/dict_ispell.c 2010-03-18 11:07:14.991697973 +0100
***************
*** 14,19 ****
--- 14,21 ----
#include "postgres.h"
#include "commands/defrem.h"
+ #include "storage/lwlock.h"
+ #include "storage/shmem.h"
#include "tsearch/dicts/spell.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_public.h"
***************
*** 21,26 ****
--- 23,29 ----
#include "utils/builtins.h"
#include "utils/memutils.h"
+ #define MAX_SHARED_DICTIONARIES 32
typedef struct
{
***************
*** 28,33 ****
--- 31,51 ----
IspellDict obj;
} DictISpell;
+ typedef struct
+ {
+ Oid oid;
+ DictISpell spell;
+ } IndexedDict;
+
+ typedef struct
+ {
+ LWLockId lock;
+ int nspells;
+ IndexedDict dicts[MAX_SHARED_DICTIONARIES];
+ } ISpellCache;
+
+ static ISpellCache *cache = NULL;
+
Datum
dispell_init(PG_FUNCTION_ARGS)
{
***************
*** 37,44 ****
dictloaded = false,
stoploaded = false;
ListCell *l;
! d = (DictISpell *) palloc0(sizeof(DictISpell));
foreach(l, dictoptions)
{
--- 55,166 ----
dictloaded = false,
stoploaded = false;
ListCell *l;
+ MemoryContext target_ctx = NULL;
+ MemoryContext work_ctx = NULL;
+ MemoryContext oldctx;
+ bool share = false;
+ Oid requested_oid = PG_GETARG_OID(1);
! /* search possible share flag */
! foreach(l, dictoptions)
! {
! DefElem *defel = (DefElem *) lfirst(l);
!
! if (pg_strcasecmp(defel->defname, "share") == 0)
! {
! if (pg_strcasecmp(defGetString(defel), "yes") == 0)
! share = true;
! else if (pg_strcasecmp(defGetString(defel), "on") == 0)
! share = true;
! else if (pg_strcasecmp(defGetString(defel), "no") == 0)
! share = false;
! else if (pg_strcasecmp(defGetString(defel), "off") == 0)
! share = false;
! else
! ereport(ERROR,
! (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
! errmsg("unsupported value of \"share\" option"),
! errhint("Use yes, no, on or off.")));
! break;
! }
! }
!
! if (share)
! {
! bool found;
! int i;
!
! if (cache == NULL)
! {
! LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
!
! /* try to take access to cache memory */
! cache = ShmemInitStruct("Shared ispell dictionaries ",
! sizeof(ISpellCache),
! &found);
! if (!cache)
! elog(ERROR, "out of shared memory");
!
! if (!found)
! {
! /* initialise cache */
! memset(cache, 0, sizeof(ISpellCache));
! cache->lock = LWLockAssign();
! }
!
! LWLockRelease(AddinShmemInitLock);
! }
!
! LWLockAcquire(cache->lock, LW_EXCLUSIVE);
!
! /*
! * try to find dictionary in cache.
! */
! for (i = 0; i < cache->nspells; i++)
! {
! if (cache->dicts[i].oid == requested_oid)
! {
! d = &(cache->dicts[i].spell);
! LWLockRelease(cache->lock);
!
! PG_RETURN_POINTER(d);
! }
! }
!
! if (cache->nspells >= MAX_SHARED_DICTIONARIES)
! elog(ERROR, "too much shered dictionaries");
!
! cache->dicts[cache->nspells].oid = requested_oid;
! d = &(cache->dicts[cache->nspells++].spell);
!
! /*
! * target is inside shared memory
! */
! target_ctx = SimpleAllocContextCreate(CurrentMemoryContext,
! "Ispell dictionary target context (simple allocator)",
! SIMPLEALLOC_LARGE_INITSIZE,
! ShmemAlloc,
! NULL);
! }
! else
! {
! /*
! * Create simple alloc context, but in process mem.
! */
! target_ctx = SimpleAllocContextCreate(CurrentMemoryContext,
! "Ispell dictionary target context (simple allocator)",
! SIMPLEALLOC_LARGE_INITSIZE,
! malloc,
! free);
! d = (DictISpell *) palloc0(sizeof(DictISpell));
! }
!
! work_ctx = AllocSetContextCreate(CurrentMemoryContext, "Ispell dictionary temporary context",
! ALLOCSET_DEFAULT_MINSIZE,
! ALLOCSET_DEFAULT_INITSIZE,
! ALLOCSET_DEFAULT_MAXSIZE);
!
! oldctx = MemoryContextSwitchTo(work_ctx);
foreach(l, dictoptions)
{
***************
*** 52,58 ****
errmsg("multiple DictFile parameters")));
NIImportDictionary(&(d->obj),
get_tsearch_config_filename(defGetString(defel),
! "dict"));
dictloaded = true;
}
else if (pg_strcasecmp(defel->defname, "AffFile") == 0)
--- 174,181 ----
errmsg("multiple DictFile parameters")));
NIImportDictionary(&(d->obj),
get_tsearch_config_filename(defGetString(defel),
! "dict"),
! target_ctx);
dictloaded = true;
}
else if (pg_strcasecmp(defel->defname, "AffFile") == 0)
***************
*** 63,69 ****
errmsg("multiple AffFile parameters")));
NIImportAffixes(&(d->obj),
get_tsearch_config_filename(defGetString(defel),
! "affix"));
affloaded = true;
}
else if (pg_strcasecmp(defel->defname, "StopWords") == 0)
--- 186,193 ----
errmsg("multiple AffFile parameters")));
NIImportAffixes(&(d->obj),
get_tsearch_config_filename(defGetString(defel),
! "affix"),
! target_ctx);
affloaded = true;
}
else if (pg_strcasecmp(defel->defname, "StopWords") == 0)
***************
*** 74,79 ****
--- 198,209 ----
errmsg("multiple StopWords parameters")));
readstoplist(defGetString(defel), &(d->stoplist), lowerstr);
stoploaded = true;
+
+ PersistStoplist(&d->stoplist, target_ctx);
+ }
+ else if (pg_strcasecmp(defel->defname, "share") == 0)
+ {
+ /* do nothing */
}
else
{
***************
*** 86,93 ****
if (affloaded && dictloaded)
{
! NISortDictionary(&(d->obj));
! NISortAffixes(&(d->obj));
}
else if (!affloaded)
{
--- 216,225 ----
if (affloaded && dictloaded)
{
! NISortDictionary(&(d->obj), target_ctx);
! NISortAffixes(&(d->obj), target_ctx);
!
! PersistAffixes(&(d->obj), target_ctx);
}
else if (!affloaded)
{
***************
*** 102,108 ****
errmsg("missing DictFile parameter")));
}
! MemoryContextDeleteChildren(CurrentMemoryContext);
PG_RETURN_POINTER(d);
}
--- 234,248 ----
errmsg("missing DictFile parameter")));
}
!
! MemoryContextSwitchTo(oldctx);
! MemoryContextDelete(work_ctx);
!
! if (share)
! {
! /* unlock share memory */
! LWLockRelease(cache->lock);
! }
PG_RETURN_POINTER(d);
}
*** ./src/backend/tsearch/spell.c.orig 2010-01-02 17:57:53.000000000 +0100
--- ./src/backend/tsearch/spell.c 2010-03-17 18:37:39.524782011 +0100
***************
*** 19,35 ****
#include "utils/memutils.h"
! /*
! * Initialization requires a lot of memory that's not needed
! * after the initialization is done. In init function,
! * CurrentMemoryContext is a long lived memory context associated
! * with the dictionary cache entry, so we use a temporary context
! * for the short-lived stuff.
! */
! static MemoryContext tmpCtx = NULL;
! #define tmpalloc(sz) MemoryContextAlloc(tmpCtx, (sz))
! #define tmpalloc0(sz) MemoryContextAllocZero(tmpCtx, (sz))
static void
checkTmpCtx(void)
--- 19,28 ----
#include "utils/memutils.h"
! static MemoryContext simple_ctx = NULL;
! #define simple_alloc(sz) MemoryContextAlloc(simple_ctx, (sz))
! #define simple_alloc0(sz) MemoryContextAllocZero(simple_ctx, (sz))
static void
checkTmpCtx(void)
***************
*** 38,66 ****
* XXX: This assumes that CurrentMemoryContext doesn't have any children
* other than the one we create here.
*/
! if (CurrentMemoryContext->firstchild == NULL)
{
! tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
! "Ispell dictionary init context",
! ALLOCSET_DEFAULT_MINSIZE,
! ALLOCSET_DEFAULT_INITSIZE,
! ALLOCSET_DEFAULT_MAXSIZE);
}
else
! tmpCtx = CurrentMemoryContext->firstchild;
! }
!
! static char *
! lowerstr_ctx(char *src)
! {
! MemoryContext saveCtx;
! char *dst;
!
! saveCtx = MemoryContextSwitchTo(tmpCtx);
! dst = lowerstr(src);
! MemoryContextSwitchTo(saveCtx);
!
! return dst;
}
#define MAX_NORM 1024
--- 31,46 ----
* XXX: This assumes that CurrentMemoryContext doesn't have any children
* other than the one we create here.
*/
! if (CurrentMemoryContext->firstchild == NULL)
{
! simple_ctx = SimpleAllocContextCreate(CurrentMemoryContext,
! "Ispell dictionary temporary context (simple allocator)",
! SIMPLEALLOC_LARGE_INITSIZE,
! malloc,
! free);
}
else
! simple_ctx = CurrentMemoryContext->firstchild;
}
#define MAX_NORM 1024
***************
*** 176,185 ****
else
{
Conf->mspell = 1024 * 20;
! Conf->Spell = (SPELL **) tmpalloc(Conf->mspell * sizeof(SPELL *));
}
}
! Conf->Spell[Conf->nspell] = (SPELL *) tmpalloc(SPELLHDRSZ + strlen(word) + 1);
strcpy(Conf->Spell[Conf->nspell]->word, word);
strncpy(Conf->Spell[Conf->nspell]->p.flag, flag, MAXFLAGLEN);
Conf->nspell++;
--- 156,165 ----
else
{
Conf->mspell = 1024 * 20;
! Conf->Spell = (SPELL **) palloc(Conf->mspell * sizeof(SPELL *));
}
}
! Conf->Spell[Conf->nspell] = (SPELL *) simple_alloc(SPELLHDRSZ + strlen(word) + 1);
strcpy(Conf->Spell[Conf->nspell]->word, word);
strncpy(Conf->Spell[Conf->nspell]->p.flag, flag, MAXFLAGLEN);
Conf->nspell++;
***************
*** 191,197 ****
* Note caller must already have applied get_tsearch_config_filename
*/
void
! NIImportDictionary(IspellDict *Conf, const char *filename)
{
tsearch_readline_state trst;
char *line;
--- 171,177 ----
* Note caller must already have applied get_tsearch_config_filename
*/
void
! NIImportDictionary(IspellDict *Conf, const char *filename, MemoryContext target_ctx)
{
tsearch_readline_state trst;
char *line;
***************
*** 242,252 ****
}
s += pg_mblen(s);
}
! pstr = lowerstr_ctx(line);
NIAddSpell(Conf, pstr, flag);
pfree(pstr);
-
pfree(line);
}
tsearch_readline_end(&trst);
--- 222,231 ----
}
s += pg_mblen(s);
}
! pstr = lowerstr(line);
NIAddSpell(Conf, pstr, flag);
pfree(pstr);
pfree(line);
}
tsearch_readline_end(&trst);
***************
*** 302,308 ****
}
static void
! NIAddAffix(IspellDict *Conf, int flag, char flagflags, const char *mask, const char *find, const char *repl, int type)
{
AFFIX *Affix;
--- 281,291 ----
}
static void
! NIAddAffix(IspellDict *Conf, int flag, char flagflags, const char *mask,
! const char *find,
! const char *repl,
! int type,
! MemoryContext target_ctx)
{
AFFIX *Affix;
***************
*** 332,338 ****
Affix->issimple = 0;
Affix->isregis = 1;
RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX) ? true : false,
! (mask && *mask) ? mask : VoidString);
}
else
{
--- 315,321 ----
Affix->issimple = 0;
Affix->isregis = 1;
RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX) ? true : false,
! (mask && *mask) ? mask : VoidString, target_ctx);
}
else
{
***************
*** 341,357 ****
int err;
pg_wchar *wmask;
char *tmask;
!
Affix->issimple = 0;
Affix->isregis = 0;
! tmask = (char *) tmpalloc(strlen(mask) + 3);
if (type == FF_SUFFIX)
sprintf(tmask, "%s$", mask);
else
sprintf(tmask, "^%s", mask);
masklen = strlen(tmask);
! wmask = (pg_wchar *) tmpalloc((masklen + 1) * sizeof(pg_wchar));
wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen);
err = pg_regcomp(&(Affix->reg.regex), wmask, wmasklen, REG_ADVANCED | REG_NOSUB);
--- 324,346 ----
int err;
pg_wchar *wmask;
char *tmask;
!
! /*
! * ToDo: Have to store compiled result to simple alloc context
! *
! * Pavel St
! */
!
Affix->issimple = 0;
Affix->isregis = 0;
! tmask = (char *) simple_alloc(strlen(mask) + 3);
if (type == FF_SUFFIX)
sprintf(tmask, "%s$", mask);
else
sprintf(tmask, "^%s", mask);
masklen = strlen(tmask);
! wmask = (pg_wchar *) simple_alloc((masklen + 1) * sizeof(pg_wchar));
wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen);
err = pg_regcomp(&(Affix->reg.regex), wmask, wmasklen, REG_ADVANCED | REG_NOSUB);
***************
*** 375,383 ****
Affix->flag = flag;
Affix->type = type;
! Affix->find = (find && *find) ? pstrdup(find) : VoidString;
if ((Affix->replen = strlen(repl)) > 0)
! Affix->repl = pstrdup(repl);
else
Affix->repl = VoidString;
Conf->naffixes++;
--- 364,372 ----
Affix->flag = flag;
Affix->type = type;
! Affix->find = (find && *find) ? MemoryContextStrdup(target_ctx,find) : VoidString;
if ((Affix->replen = strlen(repl)) > 0)
! Affix->repl = MemoryContextStrdup(target_ctx, repl);
else
Affix->repl = VoidString;
Conf->naffixes++;
***************
*** 526,532 ****
}
static void
! NIImportOOAffixes(IspellDict *Conf, const char *filename)
{
char type[BUFSIZ],
*ptype = NULL;
--- 515,521 ----
}
static void
! NIImportOOAffixes(IspellDict *Conf, const char *filename, MemoryContext target_ctx)
{
char type[BUFSIZ],
*ptype = NULL;
***************
*** 624,630 ****
if (ptype)
pfree(ptype);
! ptype = lowerstr_ctx(type);
if (scanread < 4 || (STRNCMP(ptype, "sfx") && STRNCMP(ptype, "pfx")))
goto nextline;
--- 613,619 ----
if (ptype)
pfree(ptype);
! ptype = lowerstr(type);
if (scanread < 4 || (STRNCMP(ptype, "sfx") && STRNCMP(ptype, "pfx")))
goto nextline;
***************
*** 646,652 ****
if (strlen(sflag) != 1 || flag != *sflag || flag == 0)
goto nextline;
! prepl = lowerstr_ctx(repl);
/* affix flag */
if ((ptr = strchr(prepl, '/')) != NULL)
{
--- 635,641 ----
if (strlen(sflag) != 1 || flag != *sflag || flag == 0)
goto nextline;
! prepl = lowerstr(repl);
/* affix flag */
if ((ptr = strchr(prepl, '/')) != NULL)
{
***************
*** 658,672 ****
ptr++;
}
}
! pfind = lowerstr_ctx(find);
! pmask = lowerstr_ctx(mask);
if (t_iseq(find, '0'))
*pfind = '\0';
if (t_iseq(repl, '0'))
*prepl = '\0';
NIAddAffix(Conf, flag, flagflags | aflg, pmask, pfind, prepl,
! isSuffix ? FF_SUFFIX : FF_PREFIX);
pfree(prepl);
pfree(pfind);
pfree(pmask);
--- 647,661 ----
ptr++;
}
}
! pfind = lowerstr(find);
! pmask = lowerstr(mask);
if (t_iseq(find, '0'))
*pfind = '\0';
if (t_iseq(repl, '0'))
*prepl = '\0';
NIAddAffix(Conf, flag, flagflags | aflg, pmask, pfind, prepl,
! isSuffix ? FF_SUFFIX : FF_PREFIX, target_ctx);
pfree(prepl);
pfree(pfind);
pfree(pmask);
***************
*** 687,693 ****
* Note caller must already have applied get_tsearch_config_filename
*/
void
! NIImportAffixes(IspellDict *Conf, const char *filename)
{
char *pstr = NULL;
char mask[BUFSIZ];
--- 676,682 ----
* Note caller must already have applied get_tsearch_config_filename
*/
void
! NIImportAffixes(IspellDict *Conf, const char *filename, MemoryContext target_ctx)
{
char *pstr = NULL;
char mask[BUFSIZ];
***************
*** 802,808 ****
(errcode(ERRCODE_CONFIG_FILE_ERROR),
errmsg("wrong affix file format for flag")));
tsearch_readline_end(&trst);
! NIImportOOAffixes(Conf, filename);
return;
}
if ((!suffixes) && (!prefixes))
--- 791,797 ----
(errcode(ERRCODE_CONFIG_FILE_ERROR),
errmsg("wrong affix file format for flag")));
tsearch_readline_end(&trst);
! NIImportOOAffixes(Conf, filename, target_ctx);
return;
}
if ((!suffixes) && (!prefixes))
***************
*** 811,817 ****
if (!parse_affentry(pstr, mask, find, repl))
goto nextline;
! NIAddAffix(Conf, flag, flagflags, mask, find, repl, suffixes ? FF_SUFFIX : FF_PREFIX);
nextline:
pfree(recoded);
--- 800,806 ----
if (!parse_affentry(pstr, mask, find, repl))
goto nextline;
! NIAddAffix(Conf, flag, flagflags, mask, find, repl, suffixes ? FF_SUFFIX : FF_PREFIX, target_ctx);
nextline:
pfree(recoded);
***************
*** 859,865 ****
}
static SPNode *
! mkSPNode(IspellDict *Conf, int low, int high, int level)
{
int i;
int nchar = 0;
--- 848,854 ----
}
static SPNode *
! mkSPNode(IspellDict *Conf, int low, int high, int level, MemoryContext target_ctx)
{
int i;
int nchar = 0;
***************
*** 878,884 ****
if (!nchar)
return NULL;
! rs = (SPNode *) palloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
rs->length = nchar;
data = rs->data;
--- 867,874 ----
if (!nchar)
return NULL;
! rs = (SPNode *) MemoryContextAllocZero(target_ctx,
! SPNHDRSZ + nchar * sizeof(SPNodeData));
rs->length = nchar;
data = rs->data;
***************
*** 890,896 ****
{
if (lastchar)
{
! data->node = mkSPNode(Conf, lownew, i, level + 1);
lownew = i;
data++;
}
--- 880,886 ----
{
if (lastchar)
{
! data->node = mkSPNode(Conf, lownew, i, level + 1, target_ctx);
lownew = i;
data++;
}
***************
*** 929,935 ****
}
}
! data->node = mkSPNode(Conf, lownew, high, level + 1);
return rs;
}
--- 919,925 ----
}
}
! data->node = mkSPNode(Conf, lownew, high, level + 1, target_ctx);
return rs;
}
***************
*** 939,945 ****
* and affixes.
*/
void
! NISortDictionary(IspellDict *Conf)
{
int i;
int naffix = 0;
--- 929,935 ----
* and affixes.
*/
void
! NISortDictionary(IspellDict *Conf, MemoryContext target_ctx)
{
int i;
int naffix = 0;
***************
*** 959,971 ****
if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag, MAXFLAGLEN))
naffix++;
}
!
/*
* Fill in Conf->AffixData with the affixes that were used in the
* dictionary. Replace textual flag-field of Conf->Spell entries with
* indexes into Conf->AffixData array.
*/
! Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
curaffix = -1;
for (i = 0; i < Conf->nspell; i++)
--- 949,961 ----
if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag, MAXFLAGLEN))
naffix++;
}
!
/*
* Fill in Conf->AffixData with the affixes that were used in the
* dictionary. Replace textual flag-field of Conf->Spell entries with
* indexes into Conf->AffixData array.
*/
! Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
curaffix = -1;
for (i = 0; i < Conf->nspell; i++)
***************
*** 984,996 ****
Conf->lenAffixData = Conf->nAffixData = naffix;
qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
! Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0);
Conf->Spell = NULL;
}
static AffixNode *
! mkANode(IspellDict *Conf, int low, int high, int level, int type)
{
int i;
int nchar = 0;
--- 974,986 ----
Conf->lenAffixData = Conf->nAffixData = naffix;
qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
! Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0, target_ctx);
Conf->Spell = NULL;
}
static AffixNode *
! mkANode(IspellDict *Conf, int low, int high, int level, int type, MemoryContext target_ctx)
{
int i;
int nchar = 0;
***************
*** 1011,1020 ****
if (!nchar)
return NULL;
! aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
naff = 0;
! rs = (AffixNode *) palloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
rs->length = nchar;
data = rs->data;
--- 1001,1012 ----
if (!nchar)
return NULL;
! aff = (AFFIX **) palloc(sizeof(AFFIX *) * (high - low + 1));
naff = 0;
! rs = (AffixNode *) MemoryContextAllocZero(target_ctx,
! ANHRDSZ + nchar * sizeof(AffixNodeData));
!
rs->length = nchar;
data = rs->data;
***************
*** 1026,1036 ****
{
if (lastchar)
{
! data->node = mkANode(Conf, lownew, i, level + 1, type);
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
--- 1018,1028 ----
{
if (lastchar)
{
! data->node = mkANode(Conf, lownew, i, level + 1, type, target_ctx);
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) MemoryContextAlloc(target_ctx, sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
***************
*** 1046,1056 ****
}
}
! data->node = mkANode(Conf, lownew, high, level + 1, type);
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
--- 1038,1048 ----
}
}
! data->node = mkANode(Conf, lownew, high, level + 1, type, target_ctx);
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) MemoryContextAlloc(target_ctx, sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
***************
*** 1061,1073 ****
}
static void
! mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix)
{
int i,
cnt = 0;
int start = (issuffix) ? startsuffix : 0;
int end = (issuffix) ? Conf->naffixes : startsuffix;
! AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData));
Affix->length = 1;
Affix->isvoid = 1;
--- 1053,1066 ----
}
static void
! mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix, MemoryContext target_ctx)
{
int i,
cnt = 0;
int start = (issuffix) ? startsuffix : 0;
int end = (issuffix) ? Conf->naffixes : startsuffix;
!
! AffixNode *Affix = (AffixNode *) MemoryContextAllocZero(target_ctx, ANHRDSZ + sizeof(AffixNodeData));
Affix->length = 1;
Affix->isvoid = 1;
***************
*** 1091,1097 ****
if (cnt == 0)
return;
! Affix->data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * cnt);
Affix->data->naff = (uint32) cnt;
cnt = 0;
--- 1084,1090 ----
if (cnt == 0)
return;
! Affix->data->aff = (AFFIX **) MemoryContextAllocZero(target_ctx, sizeof(AFFIX *) * cnt);
Affix->data->naff = (uint32) cnt;
cnt = 0;
***************
*** 1115,1135 ****
return false;
}
void
! NISortAffixes(IspellDict *Conf)
{
AFFIX *Affix;
size_t i;
CMPDAffix *ptr;
int firstsuffix = Conf->naffixes;
checkTmpCtx();
if (Conf->naffixes == 0)
return;
if (Conf->naffixes > 1)
qsort((void *) Conf->Affix, Conf->naffixes, sizeof(AFFIX), cmpaffix);
Conf->CompoundAffix = ptr = (CMPDAffix *) palloc(sizeof(CMPDAffix) * Conf->naffixes);
ptr->affix = NULL;
--- 1108,1163 ----
return false;
}
+ /*
+ * I would to move all static data to separate context. This context
+ * can be simply created in shared memory later. So it is step to
+ * sharing dictionaries.
+ */
+ void
+ PersistAffixes(IspellDict *Conf, MemoryContext target_ctx)
+ {
+ int i;
+ char **ptr;
+
+ /*
+ * Doesn't deallocate memory, because this stage is final,
+ * and temporary context will be dropped soon.
+ */
+ ptr = (char **) MemoryContextAlloc(target_ctx,
+ Conf->nAffixData * sizeof(char*));
+ for (i = 0; i < Conf->nAffixData; i++)
+ ptr[i] = MemoryContextStrdup(target_ctx, Conf->AffixData[i]);
+
+ Conf->AffixData = ptr;
+ }
+
void
! NISortAffixes(IspellDict *Conf, MemoryContext target_ctx)
{
AFFIX *Affix;
size_t i;
CMPDAffix *ptr;
int firstsuffix = Conf->naffixes;
+
+ CMPDAffix *buffer;
checkTmpCtx();
if (Conf->naffixes == 0)
return;
+ /*
+ * store to simple alloc context
+ */
+ Affix = (AFFIX *) MemoryContextAllocZero(target_ctx,
+ Conf->naffixes * sizeof(AFFIX));
+ memcpy(Affix, Conf->Affix, Conf->naffixes * sizeof(AFFIX));
+ pfree(Conf->Affix);
+ Conf->Affix = Affix;
+
if (Conf->naffixes > 1)
qsort((void *) Conf->Affix, Conf->naffixes, sizeof(AFFIX), cmpaffix);
+
Conf->CompoundAffix = ptr = (CMPDAffix *) palloc(sizeof(CMPDAffix) * Conf->naffixes);
ptr->affix = NULL;
***************
*** 1157,1168 ****
}
}
ptr->affix = NULL;
! Conf->CompoundAffix = (CMPDAffix *) repalloc(Conf->CompoundAffix, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1));
!
! Conf->Prefix = mkANode(Conf, 0, firstsuffix, 0, FF_PREFIX);
! Conf->Suffix = mkANode(Conf, firstsuffix, Conf->naffixes, 0, FF_SUFFIX);
! mkVoidAffix(Conf, true, firstsuffix);
! mkVoidAffix(Conf, false, firstsuffix);
}
static AffixNodeData *
--- 1185,1200 ----
}
}
ptr->affix = NULL;
!
! buffer = (CMPDAffix *) MemoryContextAllocZero(target_ctx, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1));
! memcpy(buffer, Conf->CompoundAffix, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1));
! pfree(Conf->CompoundAffix);
! Conf->CompoundAffix = buffer;
!
! Conf->Prefix = mkANode(Conf, 0, firstsuffix, 0, FF_PREFIX, target_ctx);
! Conf->Suffix = mkANode(Conf, firstsuffix, Conf->naffixes, 0, FF_SUFFIX, target_ctx);
! mkVoidAffix(Conf, true, firstsuffix, target_ctx);
! mkVoidAffix(Conf, false, firstsuffix, target_ctx);
}
static AffixNodeData *
*** ./src/backend/tsearch/ts_utils.c.orig 2010-03-17 15:49:48.248656967 +0100
--- ./src/backend/tsearch/ts_utils.c 2010-03-17 15:54:11.505781913 +0100
***************
*** 153,155 ****
--- 153,171 ----
sizeof(char *), comparestr)) ? true : false;
}
+ void
+ PersistStoplist(StopList *s, MemoryContext target_ctx)
+ {
+ char **stop;
+ int i;
+
+ stop = (char **) MemoryContextAlloc(target_ctx,
+ s->len * sizeof(char *));
+ /*
+ * Doesn't deallocate memory, because this stage is final,
+ * and temporary context will be dropped soon.
+ */
+ for (i = 0; i < s->len; i++)
+ stop[i] = MemoryContextStrdup(target_ctx, s->stop[i]);
+ s->stop = stop;
+ }
*** ./src/backend/utils/cache/ts_cache.c.orig 2010-02-14 19:42:17.000000000 +0100
--- ./src/backend/utils/cache/ts_cache.c 2010-03-18 10:45:02.419054339 +0100
***************
*** 337,344 ****
dictoptions = deserialize_deflist(opt);
entry->dictData =
! DatumGetPointer(OidFunctionCall1(template->tmplinit,
! PointerGetDatum(dictoptions)));
MemoryContextSwitchTo(oldcontext);
}
--- 337,345 ----
dictoptions = deserialize_deflist(opt);
entry->dictData =
! DatumGetPointer(OidFunctionCall2(template->tmplinit,
! PointerGetDatum(dictoptions),
! ObjectIdGetDatum(dictId)));
MemoryContextSwitchTo(oldcontext);
}
*** ./src/backend/utils/misc/guc.c.orig 2010-02-26 03:01:14.000000000 +0100
--- ./src/backend/utils/misc/guc.c 2010-03-18 07:41:44.114089845 +0100
***************
*** 390,395 ****
--- 390,397 ----
int tcp_keepalives_interval;
int tcp_keepalives_count;
+ int shared_data; /* space reserved for data inside shared memory */
+
/*
* These variables are all dummies that don't do anything, except in some
* cases provide the value for SHOW to display. The real state is elsewhere
***************
*** 1405,1410 ****
--- 1407,1422 ----
},
{
+ {"shared_data", PGC_POSTMASTER, RESOURCES_MEM,
+ gettext_noop("Sets the size of memory reserved for shared data."),
+ NULL,
+ GUC_UNIT_KB
+ },
+ &shared_data,
+ 0, 0, MAX_KILOBYTES, NULL, NULL
+ },
+
+ {
{"temp_buffers", PGC_USERSET, RESOURCES_MEM,
gettext_noop("Sets the maximum number of temporary buffers used by each session."),
NULL,
*** ./src/backend/utils/misc/postgresql.conf.sample.orig 2010-02-25 14:26:16.000000000 +0100
--- ./src/backend/utils/misc/postgresql.conf.sample 2010-03-18 07:43:58.260962487 +0100
***************
*** 108,113 ****
--- 108,115 ----
#shared_buffers = 32MB # min 128kB
# (change requires restart)
+ #shared_data = 0MB # reserved for data saved to shared memory
+ # (change requires restart)
#temp_buffers = 8MB # min 800kB
#max_prepared_transactions = 0 # zero disables the feature
# (change requires restart)
*** ./src/backend/utils/mmgr/Makefile.orig 2008-02-19 11:30:09.000000000 +0100
--- ./src/backend/utils/mmgr/Makefile 2010-03-15 11:03:16.738323659 +0100
***************
*** 12,17 ****
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
! OBJS = aset.o mcxt.o portalmem.o
include $(top_srcdir)/src/backend/common.mk
--- 12,17 ----
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
! OBJS = aset.o mcxt.o portalmem.o simple_alloc.o
include $(top_srcdir)/src/backend/common.mk
*** ./src/backend/utils/mmgr/simple_alloc.c.orig 2010-03-15 11:02:48.741323741 +0100
--- ./src/backend/utils/mmgr/simple_alloc.c 2010-03-18 11:10:26.662699831 +0100
***************
*** 0 ****
--- 1,348 ----
+ #include "postgres.h"
+
+ #include "utils/memutils.h"
+
+ typedef struct AllocBlockData *AllocBlock; /* forward reference */
+
+
+ typedef struct SimpleAllocContextData
+ {
+ MemoryContextData header; /* Standard memory-context fields */
+ AllocBlock blocks; /* head of list of standard blocks */
+ AllocBlock bigblocks; /* head of list of blocks larger than initBlockSize */
+ bool isReset;
+ /* Allocation parameters for this context */
+ Size initBlockSize;
+ void *(*external_alloc) (Size size); /* allocate memory */
+ void (*external_free) (void *); /* deallocate memory */
+ } SimpleAllocContextData;
+
+ typedef SimpleAllocContextData *SimpleAlloc;
+
+ typedef struct AllocBlockData
+ {
+ AllocBlock next;
+ Size freesize;
+ char *freeptr;
+ char data[1];
+ } AllocBlockData;
+
+ #define SimpleAllocCtxIsValid(ctx) PointerIsValid(ctx)
+
+ #define ALLOC_BLOCKHDRSZ MAXALIGN(sizeof(AllocBlockData))
+
+ static void SimpleAllocInit(MemoryContext context);
+ static void SimpleAllocReset(MemoryContext context);
+ static void SimpleAllocDelete(MemoryContext context);
+ static void *SimpleAllocAlloc(MemoryContext context, Size size);
+ static void SimpleAllocStats(MemoryContext context, int level);
+ static void *SimpleAllocRealloc(MemoryContext context, void *pointer, Size size);
+ static void SimpleAllocFree(MemoryContext context, void *pointer);
+ static bool SimpleAllocIsEmpty(MemoryContext context);
+ static Size SimpleAllocGetChunkSpace(MemoryContext context, void *pointer);
+
+ #ifdef MEMORY_CONTEXT_CHECKING
+ static void SimpleAllocCheck(MemoryContext context);
+ #endif
+
+ /*
+ * This is the virtual function table for SimpleAlloc context
+ */
+ static MemoryContextMethods SimpleAllocMethods = {
+ SimpleAllocAlloc,
+ SimpleAllocFree, /* not supported */
+ SimpleAllocRealloc, /* not supported */
+ SimpleAllocInit,
+ SimpleAllocReset,
+ SimpleAllocDelete,
+ SimpleAllocGetChunkSpace, /* not supported */
+ SimpleAllocIsEmpty,
+ SimpleAllocStats
+ #ifdef MEMORY_CONTEXT_CHECKING
+ , SimpleAllocCheck
+ #endif
+ };
+
+ #ifdef RANDOMIZE_ALLOCATED_MEMORY
+
+ /*
+ * Fill a just-allocated piece of memory with "random" data. It's not really
+ * very random, just a repeating sequence with a length that's prime. What
+ * we mainly want out of it is to have a good probability that two palloc's
+ * of the same number of bytes start out containing different data.
+ */
+ static void
+ randomize_mem(char *ptr, size_t size)
+ {
+ static int save_ctr = 1;
+ int ctr;
+
+ ctr = save_ctr;
+ while (size-- > 0)
+ {
+ *ptr++ = ctr;
+ if (++ctr > 251)
+ ctr = 1;
+ }
+ save_ctr = ctr;
+ }
+ #endif /* RANDOMIZE_ALLOCATED_MEMORY */
+
+ /*
+ * SimpleAllocContextCreate
+ *
+ *
+ */
+ MemoryContext
+ SimpleAllocContextCreate(MemoryContext parent,
+ const char *name,
+ Size initBlockSize,
+ void *(*external_alloc) (Size size),
+ void (*external_free) (void *))
+ {
+ SimpleAlloc context;
+
+ context = (SimpleAlloc) MemoryContextCreate(T_SimpleAllocContext,
+ sizeof(SimpleAllocContextData),
+ &SimpleAllocMethods,
+ parent,
+ name);
+ context->initBlockSize = MAXALIGN(initBlockSize);
+ context->external_alloc = external_alloc;
+ context->external_free = external_free;
+
+ context->blocks = NULL;
+ context->bigblocks = NULL;
+ context->isReset = true;
+
+ return (MemoryContext) context;
+ }
+
+ /*
+ * Context type methods
+ */
+ static void
+ SimpleAllocInit(MemoryContext context)
+ {
+ /*
+ * do nothing
+ */
+ }
+
+ static void
+ SimpleAllocReset(MemoryContext context)
+ {
+ AllocBlock block;
+ AllocBlock next;
+ SimpleAlloc simple_ctx = (SimpleAlloc) context;
+
+ Assert(SimpleAllocCtxIsValid(simple_ctx));
+
+ /*
+ * because simple alloc context doesn't support operation free
+ * we don't need to implement keeper block.
+ */
+ block = simple_ctx->blocks;
+ while (block != NULL)
+ {
+ next = block->next;
+ #ifdef CLOBBER_FREED_MEMORY
+ memset(block->data, 0x7F, simple_ctx->initBlockSize);
+ #endif
+
+ if (simple_ctx->external_free != NULL)
+ simple_ctx->external_free(block);
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot release allocated memory")));
+ block = next;
+ }
+
+ block = simple_ctx->bigblocks;
+ while (block != NULL)
+ {
+ next = block->next;
+
+ #ifdef CLOBBER_FREED_MEMORY
+ memset(block->data, 0x7F, simple_ctx->initBlockSize);
+ #endif
+
+ if (simple_ctx->external_free != NULL)
+ simple_ctx->external_free(block);
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot release allocated memory")));
+ block = next;
+ }
+
+ simple_ctx->isReset = true;
+ simple_ctx->blocks = NULL;
+ simple_ctx->bigblocks = NULL;
+ }
+
+ static void
+ SimpleAllocDelete(MemoryContext context)
+ {
+ SimpleAllocReset(context);
+ }
+
+ static void *
+ SimpleAllocAlloc(MemoryContext context, Size size)
+ {
+ SimpleAlloc simple_ctx = (SimpleAlloc) context;
+ AllocBlock block;
+ void *newPointer;
+ Size allocSize;
+ bool isBigBlock;
+
+ Assert(SimpleAllocCtxIsValid(simple_ctx));
+
+ /*
+ * Allocate separate block for allocation larger than 1/2 of initBlockSize
+ */
+ size = MAXALIGN(size);
+ if (size > (simple_ctx->initBlockSize/3*2))
+ {
+ /*
+ * we will allocate a big block - block only for current
+ * allocation - these blocks are is separete list.
+ */
+ allocSize = size;
+ isBigBlock = true;
+ }
+ else
+ {
+ allocSize = simple_ctx->initBlockSize;
+ isBigBlock = false;
+ }
+
+ /*
+ * Allocate a new block if there are no allocated or is big
+ * or isn't enought space.
+ */
+ if (simple_ctx->blocks == NULL || isBigBlock ||
+ (!isBigBlock && simple_ctx->blocks && simple_ctx->blocks->freesize < size))
+ {
+ /* alloc a new block */
+
+ block = simple_ctx->external_alloc(MAXALIGN(allocSize + sizeof(AllocBlockData) - 1));
+ if (block == NULL)
+ {
+ MemoryContextStats(TopMemoryContext);
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory"),
+ errdetail("Failed on request of size %lu.",
+ (unsigned long) allocSize + sizeof(AllocBlockData) - 1)));
+ }
+
+ block->freesize = allocSize;
+ block->freeptr = block->data;
+ if (!isBigBlock)
+ {
+ block->next = simple_ctx->blocks;
+ simple_ctx->blocks = block;
+ }
+ else
+ {
+ block->next = simple_ctx->bigblocks;
+ simple_ctx->bigblocks = block;
+ }
+ }
+ else
+ block = simple_ctx->blocks;
+
+ Assert(size <= block->freesize);
+ newPointer = block->freeptr;
+
+ if (!isBigBlock)
+ {
+ block->freeptr += size;
+ block->freesize -= size;
+ }
+
+ #ifdef RANDOMIZE_ALLOCATED_MEMORY
+ randomize_mem((char *) newPointer, size);
+ #endif
+ simple_ctx->isReset = false;
+
+ return newPointer;
+ }
+
+ /*
+ * SimpleAllocStats
+ */
+ static void
+ SimpleAllocStats(MemoryContext context, int level)
+ {
+ SimpleAlloc simple_ctx = (SimpleAlloc) context;
+ AllocBlock block;
+ long nblocks = 0;
+ long totalspace = 0;
+ long freespace = 0;
+ int i;
+
+ for (block = simple_ctx->blocks; block != NULL; block = block->next)
+ {
+ nblocks++;
+ totalspace += MAXALIGN(simple_ctx->initBlockSize + sizeof(AllocBlockData) - 1);
+ freespace += block->freesize;
+ }
+
+ for (block = simple_ctx->bigblocks; block != NULL; block = block->next)
+ {
+ nblocks++;
+ totalspace += block->freesize + sizeof(AllocBlockData) - 1;
+ }
+
+ for (i = 0; i < level; i++)
+ fprintf(stderr, " ");
+
+ fprintf(stderr,
+ "%s: %lu total in %ld blocks; %lu free; %lu used\n",
+ simple_ctx->header.name, totalspace, nblocks, freespace,
+ totalspace - freespace);
+ }
+
+ static void *
+ SimpleAllocRealloc(MemoryContext context, void *pointer, Size size)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("simple allocator can't realloc memory")));
+ return NULL; /* be compiler quite */
+ }
+
+ static void
+ SimpleAllocFree(MemoryContext context, void *pointer)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("simple allocator can't release memory")));
+ }
+
+ static bool
+ SimpleAllocIsEmpty(MemoryContext context)
+ {
+ return ((SimpleAlloc) context)->isReset;
+ }
+
+ static Size
+ SimpleAllocGetChunkSpace(MemoryContext context, void *pointer)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("simple allocator has not chunks")));
+ return 0; /* be compile quite */
+ }
+
+ #ifdef MEMORY_CONTEXT_CHECKING
+ static void
+ SimpleAllocCheck(MemoryContext context)
+ {
+ /* do nothing */
+ }
+
+ #endif
*** ./src/include/nodes/memnodes.h.orig 2010-01-02 17:58:04.000000000 +0100
--- ./src/include/nodes/memnodes.h 2010-03-15 14:07:38.597323708 +0100
***************
*** 72,77 ****
*/
#define MemoryContextIsValid(context) \
((context) != NULL && \
! (IsA((context), AllocSetContext)))
#endif /* MEMNODES_H */
--- 72,77 ----
*/
#define MemoryContextIsValid(context) \
((context) != NULL && \
! (IsA((context), AllocSetContext) || IsA((context), SimpleAllocContext)))
#endif /* MEMNODES_H */
*** ./src/include/nodes/nodes.h.orig 2010-01-05 22:53:59.000000000 +0100
--- ./src/include/nodes/nodes.h 2010-03-15 14:06:20.021447204 +0100
***************
*** 234,239 ****
--- 234,240 ----
*/
T_MemoryContext = 600,
T_AllocSetContext,
+ T_SimpleAllocContext,
/*
* TAGS FOR VALUE NODES (value.h)
*** ./src/include/tsearch/dicts/spell.h.orig 2010-01-02 17:58:09.000000000 +0100
--- ./src/include/tsearch/dicts/spell.h 2010-03-17 15:22:17.252656966 +0100
***************
*** 161,169 ****
} IspellDict;
extern TSLexeme *NINormalizeWord(IspellDict *Conf, char *word);
! extern void NIImportAffixes(IspellDict *Conf, const char *filename);
! extern void NIImportDictionary(IspellDict *Conf, const char *filename);
! extern void NISortDictionary(IspellDict *Conf);
! extern void NISortAffixes(IspellDict *Conf);
#endif
--- 161,171 ----
} IspellDict;
extern TSLexeme *NINormalizeWord(IspellDict *Conf, char *word);
! extern void NIImportAffixes(IspellDict *Conf, const char *filename, MemoryContext target_ctx);
! extern void NIImportDictionary(IspellDict *Conf, const char *filename, MemoryContext target_ctx);
! extern void NISortDictionary(IspellDict *Conf, MemoryContext target_ctx);
! extern void NISortAffixes(IspellDict *Conf, MemoryContext target_ctx);
!
! void PersistAffixes(IspellDict *Conf, MemoryContext target_ctx);
#endif
*** ./src/include/tsearch/ts_public.h.orig 2010-01-02 17:58:09.000000000 +0100
--- ./src/include/tsearch/ts_public.h 2010-03-17 15:22:38.692656994 +0100
***************
*** 77,82 ****
--- 77,83 ----
extern void readstoplist(const char *fname, StopList *s,
char *(*wordop) (const char *));
extern bool searchstoplist(StopList *s, char *key);
+ extern void PersistStoplist(StopList *s, MemoryContext target_ctx);
/*
* Interface with dictionaries
*** ./src/include/utils/memutils.h.orig 2010-03-15 14:33:58.000000000 +0100
--- ./src/include/utils/memutils.h 2010-03-15 16:20:24.280321906 +0100
***************
*** 136,139 ****
--- 136,147 ----
#define ALLOCSET_SMALL_INITSIZE (1 * 1024)
#define ALLOCSET_SMALL_MAXSIZE (8 * 1024)
+ extern MemoryContext SimpleAllocContextCreate(MemoryContext parent,
+ const char *name,
+ Size initBlockSize,
+ void *(*external_alloc) (Size size),
+ void (*external_free) (void *));
+
+ #define SIMPLEALLOC_LARGE_INITSIZE (1024 * 128)
+
#endif /* MEMUTILS_H */
Pavel Stehule wrote:
attached patch add possibility to share ispell dictionary between
processes. The reason for this is the slowness of first tsearch query
and size of allocated memory per process. When I tested loading of
ispell dictionary (for Czech language) I got about 500 ms and 48MB.
With simple allocator it uses only 25 MB. If we remove some check and
tolower string transformation from loading stage it needs only 200 ms.
But with broken dict or affix file it can put wrong results. This
patch significantly reduce load on servers that use ispell
dictionaries.I know so Tom worries about using of share memory. I think so it
unnecessarily. After loading data from dictionary are only read, never
modified. Second idea - this dictionary template can be distributed as
separate project (it needs a few changes in core - and simple
allocator).
Fixed-size shared memory blocks are always problematic. Would it be
possible to do the preloading with shared_preload_libraries somehow?
--
Heikki Linnakangas
EnterpriseDB http://www.enterprisedb.com
2010/3/18 Heikki Linnakangas <heikki.linnakangas@enterprisedb.com>:
Pavel Stehule wrote:
attached patch add possibility to share ispell dictionary between
processes. The reason for this is the slowness of first tsearch query
and size of allocated memory per process. When I tested loading of
ispell dictionary (for Czech language) I got about 500 ms and 48MB.
With simple allocator it uses only 25 MB. If we remove some check and
tolower string transformation from loading stage it needs only 200 ms.
But with broken dict or affix file it can put wrong results. This
patch significantly reduce load on servers that use ispell
dictionaries.I know so Tom worries about using of share memory. I think so it
unnecessarily. After loading data from dictionary are only read, never
modified. Second idea - this dictionary template can be distributed as
separate project (it needs a few changes in core - and simple
allocator).Fixed-size shared memory blocks are always problematic. Would it be
possible to do the preloading with shared_preload_libraries somehow?
Maybe. But there are some disadvantages: a) you have to copy
dictionary info to config, b) on some systems can be a problem lot of
memory per process (probably not on linux). Still you have to do some
bridge between tsearch cache and preloaded data.
Pavel
Show quoted text
--
Heikki Linnakangas
EnterpriseDB http://www.enterprisedb.com
Pavel Stehule <pavel.stehule@gmail.com> writes:
I know so Tom worries about using of share memory.
You're right, and if I have any say in the matter no patch like this
will ever go in.
What I would suggest looking into is some way of preprocessing the raw
text dictionary file into a format that can be slurped into memory
quickly. The main problem compared to the way things are done now
is that the current internal format relies heavily on pointers.
Maybe you could replace those by offsets?
regards, tom lane
2010/3/18 Tom Lane <tgl@sss.pgh.pa.us>:
Pavel Stehule <pavel.stehule@gmail.com> writes:
I know so Tom worries about using of share memory.
You're right, and if I have any say in the matter no patch like this
will ever go in.What I would suggest looking into is some way of preprocessing the raw
text dictionary file into a format that can be slurped into memory
quickly. The main problem compared to the way things are done now
is that the current internal format relies heavily on pointers.
Maybe you could replace those by offsets?
You have to maintain a new application :( There can be a new kind of bugs.
I playing with preload solution now. And I found a new issue.
I don't know why, but when I preload library with large mem like
ispell, then all next operations are ten times slower :(
[pavel@nemesis tsearch]$ psql-dev3 postgres
Timing is on.
psql-dev3 (9.0devel)
Type "help" for help.
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 0,611 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 0,277 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 0,266 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 0,348 ms
postgres=# select * from ts_debug('cs','Jmenuji se Pavel Stěhule a
bydlím ve Skalici');
alias | description | token | dictionaries |
dictionary | lexemes
-----------+-------------------+---------+---------------------------+------------------+----------------
asciiword | Word, all ASCII | Jmenuji | {preloaded_cspell,simple} |
preloaded_cspell | {jmenovat}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | se | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | Pavel | {preloaded_cspell,simple} |
preloaded_cspell | {pavel,pavla}
blank | Space symbols | | {} |
|
word | Word, all letters | Stěhule | {preloaded_cspell,simple} |
preloaded_cspell | {stěhule}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | a | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
word | Word, all letters | bydlím | {preloaded_cspell,simple} |
preloaded_cspell | {bydlet,bydlit}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | ve | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | Skalici | {preloaded_cspell,simple} |
preloaded_cspell | {skalice}
(15 rows)
Time: 24,495 ms
postgres=# select * from ts_debug('cs','Jmenuji se Pavel Stěhule a
bydlím ve Skalici');
alias | description | token | dictionaries |
dictionary | lexemes
-----------+-------------------+---------+---------------------------+------------------+----------------
asciiword | Word, all ASCII | Jmenuji | {preloaded_cspell,simple} |
preloaded_cspell | {jmenovat}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | se | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | Pavel | {preloaded_cspell,simple} |
preloaded_cspell | {pavel,pavla}
blank | Space symbols | | {} |
|
word | Word, all letters | Stěhule | {preloaded_cspell,simple} |
preloaded_cspell | {stěhule}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | a | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
word | Word, all letters | bydlím | {preloaded_cspell,simple} |
preloaded_cspell | {bydlet,bydlit}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | ve | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | Skalici | {preloaded_cspell,simple} |
preloaded_cspell | {skalice}
(15 rows)
...skipping...
alias | description | token | dictionaries |
dictionary | lexemes
-----------+-------------------+---------+---------------------------+------------------+----------------
asciiword | Word, all ASCII | Jmenuji | {preloaded_cspell,simple} |
preloaded_cspell | {jmenovat}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | se | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | Pavel | {preloaded_cspell,simple} |
preloaded_cspell | {pavel,pavla}
blank | Space symbols | | {} |
|
word | Word, all letters | Stěhule | {preloaded_cspell,simple} |
preloaded_cspell | {stěhule}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | a | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
word | Word, all letters | bydlím | {preloaded_cspell,simple} |
preloaded_cspell | {bydlet,bydlit}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | ve | {preloaded_cspell,simple} |
preloaded_cspell | {}
blank | Space symbols | | {} |
|
asciiword | Word, all ASCII | Skalici | {preloaded_cspell,simple} |
preloaded_cspell | {skalice}
(15 rows)
~
~
~
Time: 18,426 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 12,700 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 12,465 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 12,603 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 12,901 ms
postgres=# select 10;
?column?
----------
10
(1 row)
Time: 12,642 ms
When I reduce memory with simple allocator, then this issue is
removed, but it is strange.
Pavel
Show quoted text
regards, tom lane
2010/3/18 Pavel Stehule <pavel.stehule@gmail.com>:
2010/3/18 Tom Lane <tgl@sss.pgh.pa.us>:
Pavel Stehule <pavel.stehule@gmail.com> writes:
I know so Tom worries about using of share memory.
You're right, and if I have any say in the matter no patch like this
will ever go in.What I would suggest looking into is some way of preprocessing the raw
text dictionary file into a format that can be slurped into memory
quickly. The main problem compared to the way things are done now
is that the current internal format relies heavily on pointers.
Maybe you could replace those by offsets?You have to maintain a new application :( There can be a new kind of bugs.
I playing with preload solution now. And I found a new issue.
I don't know why, but when I preload library with large mem like
ispell, then all next operations are ten times slower :(
this strange issue is from very large memory context. When I don't
join tseach cached context with working context, then this issue
doesn't exists.
Datum
dpreloaddict_init(PG_FUNCTION_ARGS)
{
<------>if (prepd == NULL)
<------><------>return dispell_init(fcinfo); // use without preloading
<------>else
<------>{
<------>
<------><------>//return PointerGetDatum(prepd);
<------><------>/*.
<------><------> * Add preload context to current conntext -- when
this code is active, then I have a issue
<------><------> */
<------><------>preload_ctx->parent = CurrentMemoryContext;
<------><------>preload_ctx->nextchild = CurrentMemoryContext->firstchild;
<------><------>CurrentMemoryContext->firstchild = preload_ctx;
<------><------>
<------><------>return PointerGetDatum(prepd);
<------>}
}
Pavel
Show quoted text
When I reduce memory with simple allocator, then this issue is
removed, but it is strange.Pavel
regards, tom lane
2010/3/18 Tom Lane <tgl@sss.pgh.pa.us>:
Pavel Stehule <pavel.stehule@gmail.com> writes:
I know so Tom worries about using of share memory.
You're right, and if I have any say in the matter no patch like this
will ever go in.
I wrote second patch based on preloading. For real using it needs to
design parametrisation. It working well - on Linux. It is simple and
fast (with simple alloc). I am not sure about others systems.
Minimally it can exists as contrib module.
Pavel
Show quoted text
What I would suggest looking into is some way of preprocessing the raw
text dictionary file into a format that can be slurped into memory
quickly. The main problem compared to the way things are done now
is that the current internal format relies heavily on pointers.
Maybe you could replace those by offsets?regards, tom lane
Attachments:
preload.diffapplication/octet-stream; name=preload.diffDownload
*** ./contrib/dict_preload/dict_preload.c.orig 2010-03-18 17:00:33.281409707 +0100
--- ./contrib/dict_preload/dict_preload.c 2010-03-18 17:00:00.417407791 +0100
***************
*** 0 ****
--- 1,102 ----
+ /*-------------------------------------------------------------------------
+ *
+ * dict_preload.c
+ * preloaded dictionary - Czech
+ *
+ * Copyright (c) 2007-2010, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * $PostgreSQL: pgsql/contrib/dict_int/dict_int.c,v 1.6 2010/01/02 16:57:32 momjian Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+ #include "postgres.h"
+
+ #include "commands/defrem.h"
+ #include "fmgr.h"
+ #include "nodes/makefuncs.h"
+ #include "nodes/value.h"
+ #include "tsearch/ts_public.h"
+ #include "tsearch/ts_utils.h"
+ #include "tsearch/dicts/spell.h"
+ #include "utils/memutils.h"
+
+ PG_MODULE_MAGIC;
+
+ typedef struct
+ {
+ StopList stoplist;
+ IspellDict obj;
+ } DictISpell;
+
+ MemoryContext preload_ctx;
+
+ DictISpell *prepd = NULL;
+
+ PG_FUNCTION_INFO_V1(dpreloaddict_init);
+ Datum dpreloaddict_init(PG_FUNCTION_ARGS);
+
+ PG_FUNCTION_INFO_V1(dpreloaddict_lexize);
+ Datum dpreloaddict_lexize(PG_FUNCTION_ARGS);
+
+ void _PG_init(void);
+
+ Datum
+ dpreloaddict_init(PG_FUNCTION_ARGS)
+ {
+
+ if (prepd == NULL)
+ return dispell_init(fcinfo);
+ else
+ {
+
+ /*
+ * Add preload context to current conntext
+ */
+ preload_ctx->parent = CurrentMemoryContext;
+ preload_ctx->nextchild = CurrentMemoryContext->firstchild;
+ CurrentMemoryContext->firstchild = preload_ctx;
+
+ return PointerGetDatum(prepd);
+ }
+ }
+
+ Datum
+ dpreloaddict_lexize(PG_FUNCTION_ARGS)
+ {
+ return dispell_lexize(fcinfo);
+ }
+
+
+ void
+ _PG_init()
+ {
+ static bool inited = false;
+
+ MemoryContext oldctx;
+ List *dictoptions = NIL;
+ FunctionCallInfoData fcinfo;
+
+ if (inited)
+ return;
+
+ preload_ctx = AllocSetContextCreate(NULL, "Ispell dictionary preload context",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
+ oldctx = MemoryContextSwitchTo(preload_ctx);
+
+ dictoptions = list_make1(makeDefElem("DictFile", (Node *) makeString("czech")));
+ dictoptions = lappend(dictoptions, makeDefElem("AffFile", (Node *) makeString("czech")));
+ dictoptions = lappend(dictoptions, makeDefElem("StopWords", (Node *) makeString("czech")));
+
+ InitFunctionCallInfoData(fcinfo, NULL, 1, NULL, NULL);
+ fcinfo.arg[0] = PointerGetDatum(dictoptions);
+ fcinfo.argnull[0] = false;
+
+ prepd = (DictISpell *) DatumGetPointer(dispell_init(&fcinfo));
+
+ MemoryContextSwitchTo(oldctx);
+
+ inited = true;
+ }
*** ./contrib/dict_preload/dict_preload.sql.in.orig 2010-03-18 17:00:52.317502927 +0100
--- ./contrib/dict_preload/dict_preload.sql.in 2010-03-18 13:51:50.816472166 +0100
***************
*** 0 ****
--- 1,19 ----
+ /* $PostgreSQL: pgsql/contrib/dict_int/dict_int.sql.in,v 1.3 2007/11/13 04:24:27 momjian Exp $ */
+
+ -- Adjust this setting to control where the objects get created.
+ SET search_path = public;
+
+ CREATE OR REPLACE FUNCTION dpreloaddict_init(internal)
+ RETURNS internal
+ AS 'MODULE_PATHNAME'
+ LANGUAGE C STRICT;
+
+ CREATE OR REPLACE FUNCTION dpreloaddict_lexize(internal, internal, internal, internal)
+ RETURNS internal
+ AS 'MODULE_PATHNAME'
+ LANGUAGE C STRICT;
+
+ CREATE TEXT SEARCH TEMPLATE preloaddict_template (
+ LEXIZE = dpreloaddict_lexize,
+ INIT = dpreloaddict_init
+ );
*** ./contrib/dict_preload/uninstall_dict_preload.sql.orig 2010-03-18 17:00:58.039409567 +0100
--- ./contrib/dict_preload/uninstall_dict_preload.sql 2010-03-18 13:52:49.064472194 +0100
***************
*** 0 ****
--- 1,10 ----
+ /* $PostgreSQL: pgsql/contrib/dict_int/uninstall_dict_int.sql,v 1.3 2007/11/13 04:24:27 momjian Exp $ */
+
+ -- Adjust this setting to control where the objects get dropped.
+ SET search_path = public;
+
+ DROP TEXT SEARCH TEMPLATE preloaddict_template CASCADE;
+
+ DROP FUNCTION dpreloaddict_init(internal);
+
+ DROP FUNCTION dpreloaddict_lexize(internal,internal,internal,internal);
*** ./src/backend/tsearch/regis.c.orig 2010-01-02 17:57:53.000000000 +0100
--- ./src/backend/tsearch/regis.c 2010-03-18 16:41:03.027708600 +0100
***************
*** 71,88 ****
}
static RegisNode *
! newRegisNode(RegisNode *prev, int len)
{
RegisNode *ptr;
! ptr = (RegisNode *) palloc0(RNHDRSZ + len + 1);
if (prev)
prev->next = ptr;
return ptr;
}
void
! RS_compile(Regis *r, bool issuffix, const char *str)
{
int len = strlen(str);
int state = RS_IN_WAIT;
--- 71,89 ----
}
static RegisNode *
! newRegisNode(RegisNode *prev, int len, MemoryContext simple_ctx)
{
RegisNode *ptr;
! ptr = (RegisNode *) MemoryContextAllocZero(simple_ctx,
! RNHDRSZ + len + 1);
if (prev)
prev->next = ptr;
return ptr;
}
void
! RS_compile(Regis *r, bool issuffix, const char *str, MemoryContext simple_ctx)
{
int len = strlen(str);
int state = RS_IN_WAIT;
***************
*** 99,107 ****
if (t_isalpha(c))
{
if (ptr)
! ptr = newRegisNode(ptr, len);
else
! ptr = r->node = newRegisNode(NULL, len);
COPYCHAR(ptr->data, c);
ptr->type = RSF_ONEOF;
ptr->len = pg_mblen(c);
--- 100,108 ----
if (t_isalpha(c))
{
if (ptr)
! ptr = newRegisNode(ptr, len, simple_ctx);
else
! ptr = r->node = newRegisNode(NULL, len, simple_ctx);
COPYCHAR(ptr->data, c);
ptr->type = RSF_ONEOF;
ptr->len = pg_mblen(c);
***************
*** 109,117 ****
else if (t_iseq(c, '['))
{
if (ptr)
! ptr = newRegisNode(ptr, len);
else
! ptr = r->node = newRegisNode(NULL, len);
ptr->type = RSF_ONEOF;
state = RS_IN_ONEOF;
}
--- 110,118 ----
else if (t_iseq(c, '['))
{
if (ptr)
! ptr = newRegisNode(ptr, len, simple_ctx);
else
! ptr = r->node = newRegisNode(NULL, len, simple_ctx);
ptr->type = RSF_ONEOF;
state = RS_IN_ONEOF;
}
*** ./src/backend/utils/mmgr/Makefile.orig 2010-03-18 15:15:23.063794982 +0100
--- ./src/backend/utils/mmgr/Makefile 2010-03-18 15:15:29.040796648 +0100
***************
*** 12,17 ****
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
! OBJS = aset.o mcxt.o portalmem.o
include $(top_srcdir)/src/backend/common.mk
--- 12,17 ----
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
! OBJS = aset.o mcxt.o portalmem.o simple_alloc.o
include $(top_srcdir)/src/backend/common.mk
*** ./src/backend/utils/mmgr/simple_alloc.c.orig 2010-03-18 15:14:59.593793406 +0100
--- ./src/backend/utils/mmgr/simple_alloc.c 2010-03-18 11:10:26.000000000 +0100
***************
*** 0 ****
--- 1,348 ----
+ #include "postgres.h"
+
+ #include "utils/memutils.h"
+
+ typedef struct AllocBlockData *AllocBlock; /* forward reference */
+
+
+ typedef struct SimpleAllocContextData
+ {
+ MemoryContextData header; /* Standard memory-context fields */
+ AllocBlock blocks; /* head of list of standard blocks */
+ AllocBlock bigblocks; /* head of list of blocks larger than initBlockSize */
+ bool isReset;
+ /* Allocation parameters for this context */
+ Size initBlockSize;
+ void *(*external_alloc) (Size size); /* allocate memory */
+ void (*external_free) (void *); /* deallocate memory */
+ } SimpleAllocContextData;
+
+ typedef SimpleAllocContextData *SimpleAlloc;
+
+ typedef struct AllocBlockData
+ {
+ AllocBlock next;
+ Size freesize;
+ char *freeptr;
+ char data[1];
+ } AllocBlockData;
+
+ #define SimpleAllocCtxIsValid(ctx) PointerIsValid(ctx)
+
+ #define ALLOC_BLOCKHDRSZ MAXALIGN(sizeof(AllocBlockData))
+
+ static void SimpleAllocInit(MemoryContext context);
+ static void SimpleAllocReset(MemoryContext context);
+ static void SimpleAllocDelete(MemoryContext context);
+ static void *SimpleAllocAlloc(MemoryContext context, Size size);
+ static void SimpleAllocStats(MemoryContext context, int level);
+ static void *SimpleAllocRealloc(MemoryContext context, void *pointer, Size size);
+ static void SimpleAllocFree(MemoryContext context, void *pointer);
+ static bool SimpleAllocIsEmpty(MemoryContext context);
+ static Size SimpleAllocGetChunkSpace(MemoryContext context, void *pointer);
+
+ #ifdef MEMORY_CONTEXT_CHECKING
+ static void SimpleAllocCheck(MemoryContext context);
+ #endif
+
+ /*
+ * This is the virtual function table for SimpleAlloc context
+ */
+ static MemoryContextMethods SimpleAllocMethods = {
+ SimpleAllocAlloc,
+ SimpleAllocFree, /* not supported */
+ SimpleAllocRealloc, /* not supported */
+ SimpleAllocInit,
+ SimpleAllocReset,
+ SimpleAllocDelete,
+ SimpleAllocGetChunkSpace, /* not supported */
+ SimpleAllocIsEmpty,
+ SimpleAllocStats
+ #ifdef MEMORY_CONTEXT_CHECKING
+ , SimpleAllocCheck
+ #endif
+ };
+
+ #ifdef RANDOMIZE_ALLOCATED_MEMORY
+
+ /*
+ * Fill a just-allocated piece of memory with "random" data. It's not really
+ * very random, just a repeating sequence with a length that's prime. What
+ * we mainly want out of it is to have a good probability that two palloc's
+ * of the same number of bytes start out containing different data.
+ */
+ static void
+ randomize_mem(char *ptr, size_t size)
+ {
+ static int save_ctr = 1;
+ int ctr;
+
+ ctr = save_ctr;
+ while (size-- > 0)
+ {
+ *ptr++ = ctr;
+ if (++ctr > 251)
+ ctr = 1;
+ }
+ save_ctr = ctr;
+ }
+ #endif /* RANDOMIZE_ALLOCATED_MEMORY */
+
+ /*
+ * SimpleAllocContextCreate
+ *
+ *
+ */
+ MemoryContext
+ SimpleAllocContextCreate(MemoryContext parent,
+ const char *name,
+ Size initBlockSize,
+ void *(*external_alloc) (Size size),
+ void (*external_free) (void *))
+ {
+ SimpleAlloc context;
+
+ context = (SimpleAlloc) MemoryContextCreate(T_SimpleAllocContext,
+ sizeof(SimpleAllocContextData),
+ &SimpleAllocMethods,
+ parent,
+ name);
+ context->initBlockSize = MAXALIGN(initBlockSize);
+ context->external_alloc = external_alloc;
+ context->external_free = external_free;
+
+ context->blocks = NULL;
+ context->bigblocks = NULL;
+ context->isReset = true;
+
+ return (MemoryContext) context;
+ }
+
+ /*
+ * Context type methods
+ */
+ static void
+ SimpleAllocInit(MemoryContext context)
+ {
+ /*
+ * do nothing
+ */
+ }
+
+ static void
+ SimpleAllocReset(MemoryContext context)
+ {
+ AllocBlock block;
+ AllocBlock next;
+ SimpleAlloc simple_ctx = (SimpleAlloc) context;
+
+ Assert(SimpleAllocCtxIsValid(simple_ctx));
+
+ /*
+ * because simple alloc context doesn't support operation free
+ * we don't need to implement keeper block.
+ */
+ block = simple_ctx->blocks;
+ while (block != NULL)
+ {
+ next = block->next;
+ #ifdef CLOBBER_FREED_MEMORY
+ memset(block->data, 0x7F, simple_ctx->initBlockSize);
+ #endif
+
+ if (simple_ctx->external_free != NULL)
+ simple_ctx->external_free(block);
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot release allocated memory")));
+ block = next;
+ }
+
+ block = simple_ctx->bigblocks;
+ while (block != NULL)
+ {
+ next = block->next;
+
+ #ifdef CLOBBER_FREED_MEMORY
+ memset(block->data, 0x7F, simple_ctx->initBlockSize);
+ #endif
+
+ if (simple_ctx->external_free != NULL)
+ simple_ctx->external_free(block);
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot release allocated memory")));
+ block = next;
+ }
+
+ simple_ctx->isReset = true;
+ simple_ctx->blocks = NULL;
+ simple_ctx->bigblocks = NULL;
+ }
+
+ static void
+ SimpleAllocDelete(MemoryContext context)
+ {
+ SimpleAllocReset(context);
+ }
+
+ static void *
+ SimpleAllocAlloc(MemoryContext context, Size size)
+ {
+ SimpleAlloc simple_ctx = (SimpleAlloc) context;
+ AllocBlock block;
+ void *newPointer;
+ Size allocSize;
+ bool isBigBlock;
+
+ Assert(SimpleAllocCtxIsValid(simple_ctx));
+
+ /*
+ * Allocate separate block for allocation larger than 1/2 of initBlockSize
+ */
+ size = MAXALIGN(size);
+ if (size > (simple_ctx->initBlockSize/3*2))
+ {
+ /*
+ * we will allocate a big block - block only for current
+ * allocation - these blocks are is separete list.
+ */
+ allocSize = size;
+ isBigBlock = true;
+ }
+ else
+ {
+ allocSize = simple_ctx->initBlockSize;
+ isBigBlock = false;
+ }
+
+ /*
+ * Allocate a new block if there are no allocated or is big
+ * or isn't enought space.
+ */
+ if (simple_ctx->blocks == NULL || isBigBlock ||
+ (!isBigBlock && simple_ctx->blocks && simple_ctx->blocks->freesize < size))
+ {
+ /* alloc a new block */
+
+ block = simple_ctx->external_alloc(MAXALIGN(allocSize + sizeof(AllocBlockData) - 1));
+ if (block == NULL)
+ {
+ MemoryContextStats(TopMemoryContext);
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory"),
+ errdetail("Failed on request of size %lu.",
+ (unsigned long) allocSize + sizeof(AllocBlockData) - 1)));
+ }
+
+ block->freesize = allocSize;
+ block->freeptr = block->data;
+ if (!isBigBlock)
+ {
+ block->next = simple_ctx->blocks;
+ simple_ctx->blocks = block;
+ }
+ else
+ {
+ block->next = simple_ctx->bigblocks;
+ simple_ctx->bigblocks = block;
+ }
+ }
+ else
+ block = simple_ctx->blocks;
+
+ Assert(size <= block->freesize);
+ newPointer = block->freeptr;
+
+ if (!isBigBlock)
+ {
+ block->freeptr += size;
+ block->freesize -= size;
+ }
+
+ #ifdef RANDOMIZE_ALLOCATED_MEMORY
+ randomize_mem((char *) newPointer, size);
+ #endif
+ simple_ctx->isReset = false;
+
+ return newPointer;
+ }
+
+ /*
+ * SimpleAllocStats
+ */
+ static void
+ SimpleAllocStats(MemoryContext context, int level)
+ {
+ SimpleAlloc simple_ctx = (SimpleAlloc) context;
+ AllocBlock block;
+ long nblocks = 0;
+ long totalspace = 0;
+ long freespace = 0;
+ int i;
+
+ for (block = simple_ctx->blocks; block != NULL; block = block->next)
+ {
+ nblocks++;
+ totalspace += MAXALIGN(simple_ctx->initBlockSize + sizeof(AllocBlockData) - 1);
+ freespace += block->freesize;
+ }
+
+ for (block = simple_ctx->bigblocks; block != NULL; block = block->next)
+ {
+ nblocks++;
+ totalspace += block->freesize + sizeof(AllocBlockData) - 1;
+ }
+
+ for (i = 0; i < level; i++)
+ fprintf(stderr, " ");
+
+ fprintf(stderr,
+ "%s: %lu total in %ld blocks; %lu free; %lu used\n",
+ simple_ctx->header.name, totalspace, nblocks, freespace,
+ totalspace - freespace);
+ }
+
+ static void *
+ SimpleAllocRealloc(MemoryContext context, void *pointer, Size size)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("simple allocator can't realloc memory")));
+ return NULL; /* be compiler quite */
+ }
+
+ static void
+ SimpleAllocFree(MemoryContext context, void *pointer)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("simple allocator can't release memory")));
+ }
+
+ static bool
+ SimpleAllocIsEmpty(MemoryContext context)
+ {
+ return ((SimpleAlloc) context)->isReset;
+ }
+
+ static Size
+ SimpleAllocGetChunkSpace(MemoryContext context, void *pointer)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("simple allocator has not chunks")));
+ return 0; /* be compile quite */
+ }
+
+ #ifdef MEMORY_CONTEXT_CHECKING
+ static void
+ SimpleAllocCheck(MemoryContext context)
+ {
+ /* do nothing */
+ }
+
+ #endif
*** ./src/include/nodes/memnodes.h.orig 2010-01-02 17:58:04.000000000 +0100
--- ./src/include/nodes/memnodes.h 2010-03-18 15:18:55.349792968 +0100
***************
*** 72,77 ****
*/
#define MemoryContextIsValid(context) \
((context) != NULL && \
! (IsA((context), AllocSetContext)))
#endif /* MEMNODES_H */
--- 72,77 ----
*/
#define MemoryContextIsValid(context) \
((context) != NULL && \
! (IsA((context), AllocSetContext) || IsA((context), SimpleAllocContext)))
#endif /* MEMNODES_H */
*** ./src/include/nodes/nodes.h.orig 2010-03-18 15:21:26.328793052 +0100
--- ./src/include/nodes/nodes.h 2010-03-18 15:21:37.257794503 +0100
***************
*** 234,239 ****
--- 234,240 ----
*/
T_MemoryContext = 600,
T_AllocSetContext,
+ T_SimpleAllocContext,
/*
* TAGS FOR VALUE NODES (value.h)
*** ./src/include/tsearch/dicts/regis.h.orig 2010-03-18 16:40:41.584584472 +0100
--- ./src/include/tsearch/dicts/regis.h 2010-03-18 16:40:13.428708569 +0100
***************
*** 40,46 ****
bool RS_isRegis(const char *str);
! void RS_compile(Regis *r, bool issuffix, const char *str);
void RS_free(Regis *r);
/*returns true if matches */
--- 40,46 ----
bool RS_isRegis(const char *str);
! void RS_compile(Regis *r, bool issuffix, const char *str, MemoryContext simple_ctx);
void RS_free(Regis *r);
/*returns true if matches */
*** ./src/include/utils/memutils.h.orig 2010-01-02 17:58:10.000000000 +0100
--- ./src/include/utils/memutils.h 2010-03-18 15:20:48.889794704 +0100
***************
*** 136,139 ****
--- 136,148 ----
#define ALLOCSET_SMALL_INITSIZE (1 * 1024)
#define ALLOCSET_SMALL_MAXSIZE (8 * 1024)
+ /* simple_alloc.c */
+ extern MemoryContext SimpleAllocContextCreate(MemoryContext parent,
+ const char *name,
+ Size initBlockSize,
+ void *(*external_alloc) (Size size),
+ void (*external_free) (void *));
+
+ #define SIMPLEALLOC_LARGE_INITSIZE (1024 * 128)
+
#endif /* MEMUTILS_H */