patch: preload dictionary new version
Hello
this version has enhanced AllocSet allocator - it can use a mmap API.
Regards
Pavel Stehule
Attachments:
preload.diffapplication/octet-stream; name=preload.diffDownload
*** ./contrib/dict_preload/dict_preload.c.orig 2010-03-18 17:00:33.000000000 +0100
--- ./contrib/dict_preload/dict_preload.c 2010-04-01 14:40:58.980976853 +0200
***************
*** 0 ****
--- 1,181 ----
+ /*-------------------------------------------------------------------------
+ *
+ * dict_preload.c
+ * preloaded dictionary
+ *
+ * Copyright (c) 2007-2010, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * $PostgreSQL: pgsql/contrib/dict_preload/dict_preload.c,v 1.6 2010/01/02 16:57:32 momjian Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+ #include <sys/mman.h>
+
+ #include "postgres.h"
+
+ #include "commands/defrem.h"
+ #include "fmgr.h"
+ #include "miscadmin.h"
+ #include "nodes/makefuncs.h"
+ #include "nodes/value.h"
+ #include "tsearch/ts_public.h"
+ #include "tsearch/ts_utils.h"
+ #include "tsearch/dicts/spell.h"
+ #include "utils/guc.h"
+ #include "utils/memutils.h"
+
+ PG_MODULE_MAGIC;
+
+ char *preload_dictfile = NULL;
+ char *preload_afffile = NULL;
+ char *preload_stopwords = NULL;
+
+ typedef struct
+ {
+ StopList stoplist;
+ IspellDict obj;
+ } DictISpell;
+
+ MemoryContext preload_ctx = NULL;
+
+ DictISpell *preload_dict = NULL;
+
+ PG_FUNCTION_INFO_V1(dpreloaddict_init);
+ Datum dpreloaddict_init(PG_FUNCTION_ARGS);
+
+ PG_FUNCTION_INFO_V1(dpreloaddict_lexize);
+ Datum dpreloaddict_lexize(PG_FUNCTION_ARGS);
+
+ void _PG_init(void);
+ void _PG_fini(void);
+
+ static DictISpell *
+ load_dictionary(void)
+ {
+ List *dictopt = NIL;
+ FunctionCallInfoData fcinfo;
+ DictISpell *result;
+
+ /*
+ * read parameters for preloaded dictionary
+ */
+ if (preload_dictfile != NULL)
+ dictopt = lappend(dictopt, makeDefElem("DictFile",
+ (Node *) makeString(preload_dictfile)));
+ if (preload_afffile != NULL)
+ dictopt = lappend(dictopt, makeDefElem("AffFile",
+ (Node *) makeString(preload_afffile)));
+ if (preload_stopwords != NULL)
+ dictopt = lappend(dictopt, makeDefElem("StopWords",
+ (Node *) makeString(preload_stopwords)));
+ /*
+ * Initialise ispell dictionary
+ */
+ InitFunctionCallInfoData(fcinfo, NULL, 1, NULL, NULL);
+ fcinfo.arg[0] = PointerGetDatum(dictopt);
+ fcinfo.argnull[0] = false;
+
+ result = (DictISpell *) DatumGetPointer(dispell_init(&fcinfo));
+
+
+ MemoryContextStats(CurrentMemoryContext);
+
+ return result;
+ }
+
+ Datum
+ dpreloaddict_init(PG_FUNCTION_ARGS)
+ {
+ static bool firsttime = true;
+
+ /*
+ * dpreloaddict_init can be called more times:
+ * CREATE TEXT SEARCH DICTIONARY
+ * DROP TEXT SEARCH DICTIONARY
+ * CREATE TEXT SEARCH DICTIONARY
+ * ...
+ */
+ if (firsttime)
+ {
+ /* In this moment, dictionary have to be loaded */
+ Assert(MemoryContextIsValid(preload_ctx));
+ Assert(preload_dict != NULL);
+
+ preload_ctx = NULL;
+ firsttime = false;
+
+ return PointerGetDatum(preload_dict);
+ }
+ else
+ return PointerGetDatum(load_dictionary());
+ }
+
+ Datum
+ dpreloaddict_lexize(PG_FUNCTION_ARGS)
+ {
+ return dispell_lexize(fcinfo);
+ }
+
+ /*
+ * Module load callback
+ */
+ void
+ _PG_init()
+ {
+ MemoryContext oldctx;
+ static bool inited = false;
+ GucContext guc_ctx;
+
+ if (inited)
+ return;
+ else
+ inited = true;
+
+ guc_ctx = process_shared_preload_libraries_in_progress ?
+ PGC_POSTMASTER : PGC_SUSET;
+
+ /* Define custom GUC variables. */
+ DefineCustomStringVariable("dict_preload.dictfile",
+ "name of file of preloaded ispell dictionary",
+ NULL,
+ &preload_dictfile,
+ NULL,
+ guc_ctx, 0,
+ NULL, NULL);
+
+ /* Define custom GUC variables. */
+ DefineCustomStringVariable("dict_preload.afffile",
+ "name of file of preloaded ispell affix",
+ NULL,
+ &preload_afffile,
+ NULL,
+ guc_ctx, 0,
+ NULL, NULL);
+
+ /* Define custom GUC variables. */
+ DefineCustomStringVariable("dict_preload.stopwords",
+ "name of file of preloaded ispell stopwords",
+ NULL,
+ &preload_stopwords,
+ NULL,
+ guc_ctx, 0,
+ NULL, NULL);
+
+ /* preload dictionary */
+ Assert(preload_ctx == NULL);
+ Assert(preload_dict == NULL);
+
+ preload_ctx = MMapAllocSetContextCreate(NULL, "Ispell dictionary preload context",
+ 512 * 1024,
+ 1024 * 1024 *4,
+ 1024 * 1024 *8,
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_ANON,
+ -1);
+ oldctx = MemoryContextSwitchTo(preload_ctx);
+
+ preload_dict = load_dictionary();
+
+ MemoryContextSwitchTo(oldctx);
+ }
*** ./contrib/dict_preload/dict_preload.sql.in.orig 2010-03-18 17:00:52.000000000 +0100
--- ./contrib/dict_preload/dict_preload.sql.in 2010-03-19 08:24:33.000000000 +0100
***************
*** 0 ****
--- 1,19 ----
+ /* $PostgreSQL: pgsql/contrib/dict_int/dict_int.sql.in,v 1.3 2007/11/13 04:24:27 momjian Exp $ */
+
+ -- Adjust this setting to control where the objects get created.
+ SET search_path = public;
+
+ CREATE OR REPLACE FUNCTION dpreloaddict_init(internal)
+ RETURNS internal
+ AS 'MODULE_PATHNAME'
+ LANGUAGE C STRICT;
+
+ CREATE OR REPLACE FUNCTION dpreloaddict_lexize(internal, internal, internal, internal)
+ RETURNS internal
+ AS 'MODULE_PATHNAME'
+ LANGUAGE C STRICT;
+
+ CREATE TEXT SEARCH TEMPLATE preloaddict(
+ LEXIZE = dpreloaddict_lexize,
+ INIT = dpreloaddict_init
+ );
*** ./contrib/dict_preload/uninstall_dict_preload.sql.orig 2010-03-18 17:00:58.000000000 +0100
--- ./contrib/dict_preload/uninstall_dict_preload.sql 2010-03-18 13:52:49.000000000 +0100
***************
*** 0 ****
--- 1,10 ----
+ /* $PostgreSQL: pgsql/contrib/dict_int/uninstall_dict_int.sql,v 1.3 2007/11/13 04:24:27 momjian Exp $ */
+
+ -- Adjust this setting to control where the objects get dropped.
+ SET search_path = public;
+
+ DROP TEXT SEARCH TEMPLATE preloaddict_template CASCADE;
+
+ DROP FUNCTION dpreloaddict_init(internal);
+
+ DROP FUNCTION dpreloaddict_lexize(internal,internal,internal,internal);
*** ./src/backend/tsearch/spell.c.orig 2010-01-02 17:57:53.000000000 +0100
--- ./src/backend/tsearch/spell.c 2010-04-01 15:09:06.512848296 +0200
***************
*** 31,36 ****
--- 31,41 ----
#define tmpalloc(sz) MemoryContextAlloc(tmpCtx, (sz))
#define tmpalloc0(sz) MemoryContextAllocZero(tmpCtx, (sz))
+ static void *simple_alloc_ptr;
+ static Size simple_alloc_free;
+
+ #define SIMPLE_ALLOC_BLOCKSIZE (1024 * 1024 * 2)
+
static void
checkTmpCtx(void)
{
***************
*** 45,50 ****
--- 50,56 ----
ALLOCSET_DEFAULT_MINSIZE,
ALLOCSET_DEFAULT_INITSIZE,
ALLOCSET_DEFAULT_MAXSIZE);
+ simple_alloc_free = 0;
}
else
tmpCtx = CurrentMemoryContext->firstchild;
***************
*** 63,68 ****
--- 69,118 ----
return dst;
}
+ static void *
+ simple_alloc(Size size)
+ {
+ void *ptr;
+
+ size = MAXALIGN(size);
+
+ if (size > simple_alloc_free)
+ {
+ simple_alloc_ptr = palloc(SIMPLE_ALLOC_BLOCKSIZE);
+ simple_alloc_free = SIMPLE_ALLOC_BLOCKSIZE;
+ }
+
+ ptr = simple_alloc_ptr;
+ simple_alloc_ptr = (char *) simple_alloc_ptr + size;
+ simple_alloc_free -= size;
+
+ return ptr;
+ }
+
+ static void *
+ simple_alloc0(Size size)
+ {
+ void *ptr;
+
+ ptr = simple_alloc(size);
+ memset(ptr, 0, size);
+
+ return ptr;
+ }
+
+ static char *
+ simple_strdup(char *str)
+ {
+ char *result;
+ int len;
+
+ len = strlen(str);
+ result = simple_alloc(len + 1);
+ memcpy(result, str, len + 1);
+
+ return result;
+ }
+
#define MAX_NORM 1024
#define MAXNORMLEN 256
***************
*** 375,383 ****
Affix->flag = flag;
Affix->type = type;
! Affix->find = (find && *find) ? pstrdup(find) : VoidString;
if ((Affix->replen = strlen(repl)) > 0)
! Affix->repl = pstrdup(repl);
else
Affix->repl = VoidString;
Conf->naffixes++;
--- 425,433 ----
Affix->flag = flag;
Affix->type = type;
! Affix->find = (find && *find) ? simple_strdup(find) : VoidString;
if ((Affix->replen = strlen(repl)) > 0)
! Affix->repl = simple_strdup(repl);
else
Affix->repl = VoidString;
Conf->naffixes++;
***************
*** 833,839 ****
}
ptr = Conf->AffixData + Conf->nAffixData;
! *ptr = palloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) +
1 /* space */ + 1 /* \0 */ );
sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]);
ptr++;
--- 883,889 ----
}
ptr = Conf->AffixData + Conf->nAffixData;
! *ptr = simple_alloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) +
1 /* space */ + 1 /* \0 */ );
sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]);
ptr++;
***************
*** 878,884 ****
if (!nchar)
return NULL;
! rs = (SPNode *) palloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
rs->length = nchar;
data = rs->data;
--- 928,934 ----
if (!nchar)
return NULL;
! rs = (SPNode *) simple_alloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
rs->length = nchar;
data = rs->data;
***************
*** 974,980 ****
{
curaffix++;
Assert(curaffix < naffix);
! Conf->AffixData[curaffix] = pstrdup(Conf->Spell[i]->p.flag);
}
Conf->Spell[i]->p.d.affix = curaffix;
--- 1024,1030 ----
{
curaffix++;
Assert(curaffix < naffix);
! Conf->AffixData[curaffix] = simple_strdup(Conf->Spell[i]->p.flag);
}
Conf->Spell[i]->p.d.affix = curaffix;
***************
*** 1014,1020 ****
aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
naff = 0;
! rs = (AffixNode *) palloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
rs->length = nchar;
data = rs->data;
--- 1064,1070 ----
aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
naff = 0;
! rs = (AffixNode *) simple_alloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
rs->length = nchar;
data = rs->data;
***************
*** 1030,1036 ****
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
--- 1080,1086 ----
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
***************
*** 1050,1056 ****
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
--- 1100,1106 ----
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
***************
*** 1067,1073 ****
cnt = 0;
int start = (issuffix) ? startsuffix : 0;
int end = (issuffix) ? Conf->naffixes : startsuffix;
! AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData));
Affix->length = 1;
Affix->isvoid = 1;
--- 1117,1123 ----
cnt = 0;
int start = (issuffix) ? startsuffix : 0;
int end = (issuffix) ? Conf->naffixes : startsuffix;
! AffixNode *Affix = (AffixNode *) simple_alloc0(ANHRDSZ + sizeof(AffixNodeData));
Affix->length = 1;
Affix->isvoid = 1;
***************
*** 1091,1097 ****
if (cnt == 0)
return;
! Affix->data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * cnt);
Affix->data->naff = (uint32) cnt;
cnt = 0;
--- 1141,1147 ----
if (cnt == 0)
return;
! Affix->data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * cnt);
Affix->data->naff = (uint32) cnt;
cnt = 0;
*** ./src/backend/utils/mmgr/aset.c.orig 2010-02-26 03:01:14.000000000 +0100
--- ./src/backend/utils/mmgr/aset.c 2010-04-01 14:16:33.719848482 +0200
***************
*** 62,67 ****
--- 62,71 ----
*-------------------------------------------------------------------------
*/
+ #include <sys/mman.h>
+ #include <unistd.h>
+ #include <stdlib.h>
+
#include "postgres.h"
#include "utils/memutils.h"
***************
*** 122,127 ****
--- 126,155 ----
typedef void *AllocPointer;
/*
+ * This is the virtual table for external memory methods
+ */
+ typedef struct MemoryAccessMethods
+ {
+ void *data; /* pointer on data used for external allocator */
+ void *(*alloc) (Size size, void *data);
+ void *(*realloc) (void *ptr, Size size, void *data);
+ void (*free) (void *ptr, void *data);
+ } MemoryAccessMethods;
+
+ typedef struct MMapMemoryContextData
+ {
+ int prot;
+ int flags;
+ int fd;
+ } MMapMemoryContextData;
+
+ #define ALLOC(_ctx, size) (*_ctx->methods->alloc) (size, _ctx->methods->data)
+ #define REALLOC(_ctx, ptr, size) (*_ctx->methods->realloc) (ptr, size, _ctx->methods->data)
+ #define FREE(_ctx, ptr) (*_ctx->methods->free) (ptr, _ctx->methods->data)
+
+ #define PAGE_ALIGN(size) (((size) / getpagesize() + 1) * getpagesize())
+
+ /*
* AllocSetContext is our standard implementation of MemoryContext.
*
* Note: isReset means there is nothing for AllocSetReset to do. This is
***************
*** 143,148 ****
--- 171,177 ----
Size nextBlockSize; /* next block size to allocate */
Size allocChunkLimit; /* effective chunk size limit */
AllocBlock keeper; /* if not NULL, keep this block over resets */
+ MemoryAccessMethods *methods;
} AllocSetContext;
typedef AllocSetContext *AllocSet;
***************
*** 220,225 ****
--- 249,270 ----
static void AllocSetCheck(MemoryContext context);
#endif
+ static void *memory_alloc(Size size, void *data);
+ static void *memory_realloc(void *ptr, Size size, void *data);
+ static void memory_free(void *ptr, void *data);
+
+ static void *mmap_alloc(Size size, void *data);
+ static void *mmap_realloc(void *ptr, Size size, void *data);
+ static void mmap_free(void *ptr, void *data);
+
+ static MemoryContext
+ _AllocSetContextCreate(MemoryContext parent, const char *name,
+ Size minContextSize,
+ Size initBlockSize,
+ Size maxBlockSize,
+ MemoryAccessMethods *methods);
+
+
/*
* This is the virtual function table for AllocSet contexts.
*/
***************
*** 238,243 ****
--- 283,295 ----
#endif
};
+ static MemoryAccessMethods MemoryAccess = {
+ NULL, /* there are no special data */
+ memory_alloc,
+ memory_realloc,
+ memory_free
+ };
+
/*
* Table for AllocSetFreeIndex
*/
***************
*** 266,271 ****
--- 318,324 ----
#define AllocAllocInfo(_cxt, _chunk)
#endif
+
/* ----------
* AllocSetFreeIndex -
*
***************
*** 353,358 ****
--- 406,427 ----
Size initBlockSize,
Size maxBlockSize)
{
+ return _AllocSetContextCreate(parent, name,
+ minContextSize,
+ initBlockSize,
+ maxBlockSize,
+ &MemoryAccess);
+ }
+
+
+ static MemoryContext
+ _AllocSetContextCreate(MemoryContext parent,
+ const char *name,
+ Size minContextSize,
+ Size initBlockSize,
+ Size maxBlockSize,
+ MemoryAccessMethods *methods)
+ {
AllocSet context;
/* Do the type-independent part of context creation */
***************
*** 376,381 ****
--- 445,451 ----
context->initBlockSize = initBlockSize;
context->maxBlockSize = maxBlockSize;
context->nextBlockSize = initBlockSize;
+ context->methods = methods;
/*
* Compute the allocation chunk size limit for this context. It can't be
***************
*** 399,405 ****
Size blksize = MAXALIGN(minContextSize);
AllocBlock block;
! block = (AllocBlock) malloc(blksize);
if (block == NULL)
{
MemoryContextStats(TopMemoryContext);
--- 469,475 ----
Size blksize = MAXALIGN(minContextSize);
AllocBlock block;
! block = (AllocBlock) ALLOC(context, blksize);
if (block == NULL)
{
MemoryContextStats(TopMemoryContext);
***************
*** 503,509 ****
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
#endif
! free(block);
}
block = next;
}
--- 573,579 ----
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
#endif
! FREE(set, block);
}
block = next;
}
***************
*** 578,584 ****
{
chunk_size = MAXALIGN(size);
blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
! block = (AllocBlock) malloc(blksize);
if (block == NULL)
{
MemoryContextStats(TopMemoryContext);
--- 648,654 ----
{
chunk_size = MAXALIGN(size);
blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
! block = (AllocBlock) ALLOC(set, blksize);
if (block == NULL)
{
MemoryContextStats(TopMemoryContext);
***************
*** 748,754 ****
blksize <<= 1;
/* Try to allocate it */
! block = (AllocBlock) malloc(blksize);
/*
* We could be asking for pretty big blocks here, so cope if malloc
--- 818,824 ----
blksize <<= 1;
/* Try to allocate it */
! block = (AllocBlock) ALLOC(set, blksize);
/*
* We could be asking for pretty big blocks here, so cope if malloc
***************
*** 759,765 ****
blksize >>= 1;
if (blksize < required_size)
break;
! block = (AllocBlock) malloc(blksize);
}
if (block == NULL)
--- 829,835 ----
blksize >>= 1;
if (blksize < required_size)
break;
! block = (AllocBlock) ALLOC(set, blksize);
}
if (block == NULL)
***************
*** 870,876 ****
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
#endif
! free(block);
}
else
{
--- 940,946 ----
/* Wipe freed memory for debugging purposes */
memset(block, 0x7F, block->freeptr - ((char *) block));
#endif
! FREE(set, block);
}
else
{
***************
*** 967,973 ****
/* Do the realloc */
chksize = MAXALIGN(size);
blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
! block = (AllocBlock) realloc(block, blksize);
if (block == NULL)
{
MemoryContextStats(TopMemoryContext);
--- 1037,1043 ----
/* Do the realloc */
chksize = MAXALIGN(size);
blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
! block = (AllocBlock) REALLOC(set, block, blksize);
if (block == NULL)
{
MemoryContextStats(TopMemoryContext);
***************
*** 1199,1201 ****
--- 1269,1418 ----
}
#endif /* MEMORY_CONTEXT_CHECKING */
+
+ /*
+ * Methods for access to glibc space
+ */
+ static void *
+ memory_alloc(Size size, void *data)
+ {
+ return malloc(size);
+ }
+
+ static void *
+ memory_realloc(void *ptr, Size size, void *data)
+ {
+ return realloc(ptr, size);
+ }
+
+ static void
+ memory_free(void *ptr, void *data)
+ {
+ free(ptr);
+ }
+
+
+ /*
+ * Methods for access to mmap space
+ */
+ MemoryContext
+ MMapAllocSetContextCreate(MemoryContext parent,
+ const char *name,
+ Size minContextSize,
+ Size initBlockSize,
+ Size maxBlockSize,
+ int prot,
+ int flags,
+ int fd)
+ {
+ MMapMemoryContextData *mmap_data;
+ MemoryAccessMethods *mmap_access;
+
+ mmap_data = MemoryContextAlloc(TopMemoryContext,
+ sizeof(MMapMemoryContextData));
+ mmap_data->prot = prot;
+ mmap_data->flags = flags;
+ mmap_data->fd = fd;
+
+ mmap_access = MemoryContextAlloc(TopMemoryContext,
+ sizeof(MemoryAccessMethods));
+
+ mmap_access->data = mmap_data;
+ mmap_access->alloc = mmap_alloc;
+ mmap_access->realloc = mmap_realloc;
+ mmap_access->free = mmap_free;
+
+ return _AllocSetContextCreate(parent, name,
+ minContextSize,
+ initBlockSize,
+ maxBlockSize,
+ mmap_access);
+ }
+
+ static void *
+ mmap_alloc(Size size, void *data)
+ {
+ void *ptr;
+ MMapMemoryContextData *mmdata = (MMapMemoryContextData *) data;
+
+ Assert(mmdata != NULL);
+
+ size = PAGE_ALIGN(MAXALIGN(size) + MAXALIGN(sizeof(size)));
+
+ ptr = mmap(NULL, size, mmdata->prot, mmdata->flags,
+ -1, 0);
+
+ if (ptr == MAP_FAILED)
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory"),
+ errdetail("Failed on request of size %lu.",
+ (unsigned long) size)));
+
+ *((Size*) ptr) = size;
+ return (char *) ptr + MAXALIGN(sizeof(Size));
+ }
+
+ static void *
+ mmap_realloc(void *ptr, Size size, void *data)
+ {
+ MMapMemoryContextData *mmdata = (MMapMemoryContextData *) data;
+ Size oldsize;
+ void *result = ptr;
+
+ ptr = (char *) ptr - MAXALIGN(sizeof(Size));
+
+ oldsize = *((Size *) ptr);
+ size = PAGE_ALIGN(MAXALIGN(size) + MAXALIGN(sizeof(Size)));
+
+ Assert(mmdata != NULL);
+
+ /*
+ * Attention - on linux you cannot call mremap together with
+ * shared
+ *
+ * ptr = mremap(ptr, oldsize,
+ * size,
+ * MREMAP_MAYMOVE);
+ */
+ if (size > oldsize)
+ {
+ void *newptr;
+ Size newsize = oldsize;
+
+ while (newsize < size)
+ newsize = PAGE_ALIGN(2 * newsize);
+
+ newptr = mmap(NULL, newsize,
+ mmdata->prot, mmdata->flags, mmdata->fd,
+ 0);
+
+ if (newptr == MAP_FAILED)
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory"),
+ errdetail("Failed on request of size %lu.",
+ (unsigned long) newsize)));
+ memcpy(newptr, ptr, oldsize);
+ *((Size*) newptr) = newsize;
+
+ munmap(ptr, oldsize);
+ result = (char *) newptr + MAXALIGN(sizeof(Size));
+ }
+
+ return result;
+ }
+
+ static void
+ mmap_free(void *ptr, void *data)
+ {
+ Size size;
+ MMapMemoryContextData *mmdata = (MMapMemoryContextData *) data;
+
+ Assert(mmdata != NULL);
+
+ ptr = (char *) ptr - MAXALIGN(sizeof(Size));
+ size = *((Size*) ptr);
+
+ munmap(ptr, size);
+ }
*** ./src/include/utils/memutils.h.orig 2010-01-02 17:58:10.000000000 +0100
--- ./src/include/utils/memutils.h 2010-04-01 14:18:33.319326916 +0200
***************
*** 120,125 ****
--- 120,134 ----
Size initBlockSize,
Size maxBlockSize);
+ extern MemoryContext MMapAllocSetContextCreate(MemoryContext parent,
+ const char *name,
+ Size minContextSize,
+ Size initBlockSize,
+ Size maxBlockSize,
+ int prot,
+ int flags,
+ int fd);
+
/*
* Recommended default alloc parameters, suitable for "ordinary" contexts
* that might hold quite a lot of data.
Pavel Stehule <pavel.stehule@gmail.com> wrote:
this version has enhanced AllocSet allocator - it can use a mmap API.
I review your patch and will report some comments. However, I don't have
test cases for the patch because there is no large dictionaries in the
default postgres installation. I'd like to ask you to supply test data
for the patch.
This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.
The dictinary loader is also modified a bit to use simple_alloc() instead
of palloc() for long-lived cache. It can reduce calls of AllocSetAlloc(),
that have some overheads to support pfree(). Since the cache is never
released, simple_alloc() seems to have better performance than palloc().
Note that the optimization will also work for non-preloaded dicts.
=== Questions ===
- How do backends share the dict cache? You might expect postmaster's
catalog is inherited to backends with fork(), but we don't use fork()
on Windows.
- Why are SQL functions dpreloaddict_init() and dpreloaddict_lexize()
defined but not used?
=== Design ===
- You added 3 custom parameters (dict_preload.dictfile/afffile/stopwords),
but I think text search configuration names is better than file names.
However, it requires system catalog access but we cannot access any
catalog at the moment of preloading. If config-name-based setting is
difficult, we need to write docs about where we can get the dict names
to be preloaded instead. (from \dFd+ ?)
- Do we need to support multiple preloaded dicts? I think dict_preload.*
should accept a list of items to be loaded. GUC_LIST_INPUT will be a help.
- Server doesn't start when I added dict_preload to
shared_preload_libraries and didn't add any custom parameters.
FATAL: missing AffFile parameter
But server should start with no effects or print WARNING messages
for "no dicts are preloaded" in such case.
- We could replace simple_alloc() to a new MemoryContextMethods that
doesn't support pfree() but has better performance. It doesn't look
ideal for me to implement simple_alloc() on the top of palloc().
=== Implementation ===
I'm sure that your patch is WIP, but I'll note some issues just in case.
- We need Makefile for contrib/dict_preload.
- mmap() is not always portable. We should check the availability
in configure, and also have an alternative implementation for Win32.
Regards,
---
Takahiro Itagaki
NTT Open Source Software Center
Hello
2010/7/8 Takahiro Itagaki <itagaki.takahiro@oss.ntt.co.jp>:
Pavel Stehule <pavel.stehule@gmail.com> wrote:
this version has enhanced AllocSet allocator - it can use a mmap API.
I review your patch and will report some comments. However, I don't have
test cases for the patch because there is no large dictionaries in the
default postgres installation. I'd like to ask you to supply test data
for the patch.
you can use a Czech dictionary - please, download it from
http://www.pgsql.cz/data/czech.tar.gz
CREATE TEXT SEARCH DICTIONARY cspell
(template=ispell, dictfile = czech, afffile=czech, stopwords=czech);
CREATE TEXT SEARCH CONFIGURATION cs (copy=english);
ALTER TEXT SEARCH CONFIGURATION cs
ALTER MAPPING FOR word, asciiword WITH cspell, simple;
postgres=# select * from ts_debug('cs','Příliš žluťoučký kůň se napil
žluté vody');
alias | description | token | dictionaries |
dictionary | lexemes
-----------+-------------------+-----------+-----------------+------------+-------------
word | Word, all letters | Příliš | {cspell,simple} | cspell
| {příliš}
blank | Space symbols | | {} | |
word | Word, all letters | žluťoučký | {cspell,simple} | cspell
| {žluťoučký}
blank | Space symbols | | {} | |
word | Word, all letters | kůň | {cspell,simple} | cspell
| {kůň}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | se | {cspell,simple} | cspell | {}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | napil | {cspell,simple} | cspell
| {napít}
blank | Space symbols | | {} | |
word | Word, all letters | žluté | {cspell,simple} | cspell
| {žlutý}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | vody | {cspell,simple} | cspell
| {voda}
This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.The dictinary loader is also modified a bit to use simple_alloc() instead
of palloc() for long-lived cache. It can reduce calls of AllocSetAlloc(),
that have some overheads to support pfree(). Since the cache is never
released, simple_alloc() seems to have better performance than palloc().
Note that the optimization will also work for non-preloaded dicts.
it produce little bit better spead, but mainly it significant memory
reduction - palloc allocation is expensive, because add 4 bytes (8
bytes) to any allocations. And it is problem for thousands smalls
blocks like TSearch ispell dictionary uses. On 64 bit the overhead is
horrible
=== Questions ===
- How do backends share the dict cache? You might expect postmaster's
catalog is inherited to backends with fork(), but we don't use fork()
on Windows.
I though about some variants
a) using a shared memory - but it needs more shared memory
reservation, maybe some GUC - but this variant was refused in
discussion.
b) using a mmap on Unix and CreateFileMapping API on windows - but it
is little bit problem for me. I am not have a develop tools for ms
windows. And I don't understand to MS Win platform :(
Magnus, can you do some tip?
Without MSWindows we don't need to solve a shared memory and can use
only fork. If we can think about MSWin too, then we have to calculate
only with some shared memory based solution. But it has more
possibilities - shared dictionary can be loaded in runtime too.
- Why are SQL functions dpreloaddict_init() and dpreloaddict_lexize()
defined but not used?
it is used, if I remember well. It uses ispell dictionary API. The
using is simlyfied - you can parametrize preload dictionary - and then
you use a preloaded dictionary - not some specific dictionary. This
has one advantage and one disadvantage + very simple configuration, +
there are not necessary some shared dictionary manager, - only one
preload dictionary can be used.
=== Design ===
- You added 3 custom parameters (dict_preload.dictfile/afffile/stopwords),
but I think text search configuration names is better than file names.
However, it requires system catalog access but we cannot access any
catalog at the moment of preloading. If config-name-based setting is
difficult, we need to write docs about where we can get the dict names
to be preloaded instead. (from \dFd+ ?)
yes - it is true argument - there are not possible access to these
data in preloaded time. I would to support preloading - (and possible
support sharing session loaded dictionaries), because it ensure a
constant time for TSearch queries everytime. Yes, some documentation,
some enhancing of dictionary list info can be solution.
- Do we need to support multiple preloaded dicts? I think dict_preload.*
should accept a list of items to be loaded. GUC_LIST_INPUT will be a help.
maybe yes. Personaly I would not to complicate a design and using. And
I don't know about request for multiple preloaded dicts now. The
preloaded dictionaries interface is only server side matter - so it
can be changed/enhanced later without problems. I have a idea about
enhancig a GUC parser to allow some like
preload_dictionary.patch = ...
preload_dictionary.czech = (template=ispell, dictfile = czech,
afffile=czech, stopwords=czech)
proload_dictionary.japan = (template=.....
- Server doesn't start when I added dict_preload to
shared_preload_libraries and didn't add any custom parameters.
FATAL: missing AffFile parameter
But server should start with no effects or print WARNING messages
for "no dicts are preloaded" in such case.- We could replace simple_alloc() to a new MemoryContextMethods that
doesn't support pfree() but has better performance. It doesn't look
ideal for me to implement simple_alloc() on the top of palloc().
I don't agree. palloc API is designed to be general - so I implemented
a new memory context type via MMapAllocSetContextCreate and then I use
a palloc function. There isn't reason to design a some new API.
=== Implementation ===
I'm sure that your patch is WIP, but I'll note some issues just in case.- We need Makefile for contrib/dict_preload.
sure, sorry
- mmap() is not always portable. We should check the availability
in configure, and also have an alternative implementation for Win32.
yes, it have to be first step. I need a established API for simple
allocation. Maybe divide this patch to two independent patches - and
to solve memory allocation first ? Dictionary preloading isn't complex
or large feature - so it can be handled in every commitfest. Memory
management is more importal, and can be handled first.
Regards,
---
Takahiro Itagaki
NTT Open Source Software Center
Thank You very much for review
Pavel Stehule
Show quoted text
Hello
I found a page http://www.genesys-e.org/jwalter//mix4win.htm where is
section >>Emulation of mmap/munmap<<. Can be a solution?
Regards
Pavel Stehule
2010/7/8 Pavel Stehule <pavel.stehule@gmail.com>:
Show quoted text
Hello
2010/7/8 Takahiro Itagaki <itagaki.takahiro@oss.ntt.co.jp>:
Pavel Stehule <pavel.stehule@gmail.com> wrote:
this version has enhanced AllocSet allocator - it can use a mmap API.
I review your patch and will report some comments. However, I don't have
test cases for the patch because there is no large dictionaries in the
default postgres installation. I'd like to ask you to supply test data
for the patch.you can use a Czech dictionary - please, download it from
http://www.pgsql.cz/data/czech.tar.gzCREATE TEXT SEARCH DICTIONARY cspell
(template=ispell, dictfile = czech, afffile=czech, stopwords=czech);
CREATE TEXT SEARCH CONFIGURATION cs (copy=english);
ALTER TEXT SEARCH CONFIGURATION cs
ALTER MAPPING FOR word, asciiword WITH cspell, simple;postgres=# select * from ts_debug('cs','Příliš žluťoučký kůň se napil
žluté vody');
alias | description | token | dictionaries |
dictionary | lexemes
-----------+-------------------+-----------+-----------------+------------+-------------
word | Word, all letters | Příliš | {cspell,simple} | cspell
| {příliš}
blank | Space symbols | | {} | |
word | Word, all letters | žluťoučký | {cspell,simple} | cspell
| {žluťoučký}
blank | Space symbols | | {} | |
word | Word, all letters | kůň | {cspell,simple} | cspell
| {kůň}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | se | {cspell,simple} | cspell | {}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | napil | {cspell,simple} | cspell
| {napít}
blank | Space symbols | | {} | |
word | Word, all letters | žluté | {cspell,simple} | cspell
| {žlutý}
blank | Space symbols | | {} | |
asciiword | Word, all ASCII | vody | {cspell,simple} | cspell
| {voda}This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.The dictinary loader is also modified a bit to use simple_alloc() instead
of palloc() for long-lived cache. It can reduce calls of AllocSetAlloc(),
that have some overheads to support pfree(). Since the cache is never
released, simple_alloc() seems to have better performance than palloc().
Note that the optimization will also work for non-preloaded dicts.it produce little bit better spead, but mainly it significant memory
reduction - palloc allocation is expensive, because add 4 bytes (8
bytes) to any allocations. And it is problem for thousands smalls
blocks like TSearch ispell dictionary uses. On 64 bit the overhead is
horrible=== Questions ===
- How do backends share the dict cache? You might expect postmaster's
catalog is inherited to backends with fork(), but we don't use fork()
on Windows.I though about some variants
a) using a shared memory - but it needs more shared memory
reservation, maybe some GUC - but this variant was refused in
discussion.
b) using a mmap on Unix and CreateFileMapping API on windows - but it
is little bit problem for me. I am not have a develop tools for ms
windows. And I don't understand to MS Win platform :(Magnus, can you do some tip?
Without MSWindows we don't need to solve a shared memory and can use
only fork. If we can think about MSWin too, then we have to calculate
only with some shared memory based solution. But it has more
possibilities - shared dictionary can be loaded in runtime too.- Why are SQL functions dpreloaddict_init() and dpreloaddict_lexize()
defined but not used?it is used, if I remember well. It uses ispell dictionary API. The
using is simlyfied - you can parametrize preload dictionary - and then
you use a preloaded dictionary - not some specific dictionary. This
has one advantage and one disadvantage + very simple configuration, +
there are not necessary some shared dictionary manager, - only one
preload dictionary can be used.=== Design ===
- You added 3 custom parameters (dict_preload.dictfile/afffile/stopwords),
but I think text search configuration names is better than file names.
However, it requires system catalog access but we cannot access any
catalog at the moment of preloading. If config-name-based setting is
difficult, we need to write docs about where we can get the dict names
to be preloaded instead. (from \dFd+ ?)yes - it is true argument - there are not possible access to these
data in preloaded time. I would to support preloading - (and possible
support sharing session loaded dictionaries), because it ensure a
constant time for TSearch queries everytime. Yes, some documentation,
some enhancing of dictionary list info can be solution.- Do we need to support multiple preloaded dicts? I think dict_preload.*
should accept a list of items to be loaded. GUC_LIST_INPUT will be a help.maybe yes. Personaly I would not to complicate a design and using. And
I don't know about request for multiple preloaded dicts now. The
preloaded dictionaries interface is only server side matter - so it
can be changed/enhanced later without problems. I have a idea about
enhancig a GUC parser to allow some likepreload_dictionary.patch = ...
preload_dictionary.czech = (template=ispell, dictfile = czech,
afffile=czech, stopwords=czech)
proload_dictionary.japan = (template=.....- Server doesn't start when I added dict_preload to
shared_preload_libraries and didn't add any custom parameters.
FATAL: missing AffFile parameter
But server should start with no effects or print WARNING messages
for "no dicts are preloaded" in such case.- We could replace simple_alloc() to a new MemoryContextMethods that
doesn't support pfree() but has better performance. It doesn't look
ideal for me to implement simple_alloc() on the top of palloc().I don't agree. palloc API is designed to be general - so I implemented
a new memory context type via MMapAllocSetContextCreate and then I use
a palloc function. There isn't reason to design a some new API.=== Implementation ===
I'm sure that your patch is WIP, but I'll note some issues just in case.- We need Makefile for contrib/dict_preload.
sure, sorry
- mmap() is not always portable. We should check the availability
in configure, and also have an alternative implementation for Win32.yes, it have to be first step. I need a established API for simple
allocation. Maybe divide this patch to two independent patches - and
to solve memory allocation first ? Dictionary preloading isn't complex
or large feature - so it can be handled in every commitfest. Memory
management is more importal, and can be handled first.Regards,
---
Takahiro Itagaki
NTT Open Source Software CenterThank You very much for review
Pavel Stehule
On Wed, Jul 7, 2010 at 10:50 PM, Takahiro Itagaki
<itagaki.takahiro@oss.ntt.co.jp> wrote:
This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.
I thought someone (Tom?) had proposed idea previously of writing a
dictionary precompiler that would produce a file which could then be
mmap()'d into the backend. Has any thought been given to that
approach?
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise Postgres Company
2010/7/8 Robert Haas <robertmhaas@gmail.com>:
On Wed, Jul 7, 2010 at 10:50 PM, Takahiro Itagaki
<itagaki.takahiro@oss.ntt.co.jp> wrote:This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.I thought someone (Tom?) had proposed idea previously of writing a
dictionary precompiler that would produce a file which could then be
mmap()'d into the backend. Has any thought been given to that
approach?
The precompiler can save only some time related to parsing. But it
isn't main issue. Without simple allocation the data from dictionary
takes about 55 MB, with simple allocation about 10 MB. If you have a
100 max_session, then these data can be 100 x repeated in memory -
about 1G (for Czech dictionary). I think so memory can be used
better.
Minimally you have to read these 10MB from disc - maybe from file
cache - but it takes some time too - but it will be significantly
better than now.
Regards
Pavel Stehule
Show quoted text
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise Postgres Company
On Thu, Jul 8, 2010 at 7:03 AM, Pavel Stehule <pavel.stehule@gmail.com> wrote:
2010/7/8 Robert Haas <robertmhaas@gmail.com>:
On Wed, Jul 7, 2010 at 10:50 PM, Takahiro Itagaki
<itagaki.takahiro@oss.ntt.co.jp> wrote:This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.I thought someone (Tom?) had proposed idea previously of writing a
dictionary precompiler that would produce a file which could then be
mmap()'d into the backend. Has any thought been given to that
approach?The precompiler can save only some time related to parsing. But it
isn't main issue. Without simple allocation the data from dictionary
takes about 55 MB, with simple allocation about 10 MB. If you have a
100 max_session, then these data can be 100 x repeated in memory -
about 1G (for Czech dictionary). I think so memory can be used
better.
A precompiler can give you all the same memory management benefits.
Minimally you have to read these 10MB from disc - maybe from file
cache - but it takes some time too - but it will be significantly
better than now.
If you use mmap(), you don't need to anything of the sort. And the
EXEC_BACKEND case doesn't require as many gymnastics, either. And the
variable can be PGC_SIGHUP or even PGC_USERSET instead of
PGC_POSTMASTER.
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise Postgres Company
2010/7/8 Robert Haas <robertmhaas@gmail.com>:
On Thu, Jul 8, 2010 at 7:03 AM, Pavel Stehule <pavel.stehule@gmail.com> wrote:
2010/7/8 Robert Haas <robertmhaas@gmail.com>:
On Wed, Jul 7, 2010 at 10:50 PM, Takahiro Itagaki
<itagaki.takahiro@oss.ntt.co.jp> wrote:This patch allocates memory with non-file-based mmap() to preload text search
dictionary files at the server start. Note that dist files are not mmap'ed
directly in the patch; mmap() is used for reallocatable shared memory.I thought someone (Tom?) had proposed idea previously of writing a
dictionary precompiler that would produce a file which could then be
mmap()'d into the backend. Has any thought been given to that
approach?The precompiler can save only some time related to parsing. But it
isn't main issue. Without simple allocation the data from dictionary
takes about 55 MB, with simple allocation about 10 MB. If you have a
100 max_session, then these data can be 100 x repeated in memory -
about 1G (for Czech dictionary). I think so memory can be used
better.A precompiler can give you all the same memory management benefits.
Minimally you have to read these 10MB from disc - maybe from file
cache - but it takes some time too - but it will be significantly
better than now.If you use mmap(), you don't need to anything of the sort. And the
EXEC_BACKEND case doesn't require as many gymnastics, either. And the
variable can be PGC_SIGHUP or even PGC_USERSET instead of
PGC_POSTMASTER.
I use mmap(). And with mmap the precompiler are not necessary.
Dictionary is loaded only one time - in original ispell format. I
think, it is much more simple for administration - just copy ispell
files. There are not some possible problems with binary
incompatibility, you don't need to solve serialisation,
deserialiasation, ...you don't need to copy TSearch ispell parser code
to client application - probably we would to support not compiled
ispell dictionaries still. Using a precompiler means a new questions
for upgrade!
The real problem is using a some API on MS Windows, where mmap doesn't exist.
I think we can divide this problem to three parts
a) simple allocator - it can be used not only for TSearch dictionaries.
b) sharing a data - it is important for large dictionaries
c) preloading - it decrease load time of first TSearch query
Regards
Pavel Stehule
Show quoted text
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise Postgres Company
Pavel Stehule <pavel.stehule@gmail.com> writes:
2010/7/8 Robert Haas <robertmhaas@gmail.com>:
A precompiler can give you all the same memory management benefits.
I use mmap(). And with mmap the precompiler are not necessary.
Dictionary is loaded only one time - in original ispell format. I
think, it is much more simple for administration - just copy ispell
files. There are not some possible problems with binary
incompatibility, you don't need to solve serialisation,
deserialiasation, ...you don't need to copy TSearch ispell parser code
to client application - probably we would to support not compiled
ispell dictionaries still. Using a precompiler means a new questions
for upgrade!
You're inventing a bunch of straw men to attack. There's no reason that
a precompiler approach would have to put any new requirements on the
user. For example, the dictionary-load code could automatically execute
the precompile step if it observed that the precompiled copy of the
dictionary was missing or had an older file timestamp than the source.
I like the idea of a precompiler step mainly because it still gives you
most of the benefits of the patch on platforms without mmap. (Instead
of mmap'ing, just open and read() the precompiled file.) In particular,
you would still have a creditable improvement for Windows users without
writing any Windows-specific code.
I think we can divide this problem to three parts
a) simple allocator - it can be used not only for TSearch dictionaries.
I think that's a waste of time, frankly. There aren't enough potential
use cases.
b) sharing a data - it is important for large dictionaries
Useful but not really essential.
c) preloading - it decrease load time of first TSearch query
This is the part that is the make-or-break benefit of the patch.
You need a solution that cuts load time even when mmap isn't
available.
regards, tom lane
2010/7/8 Tom Lane <tgl@sss.pgh.pa.us>:
Pavel Stehule <pavel.stehule@gmail.com> writes:
2010/7/8 Robert Haas <robertmhaas@gmail.com>:
A precompiler can give you all the same memory management benefits.
I use mmap(). And with mmap the precompiler are not necessary.
Dictionary is loaded only one time - in original ispell format. I
think, it is much more simple for administration - just copy ispell
files. There are not some possible problems with binary
incompatibility, you don't need to solve serialisation,
deserialiasation, ...you don't need to copy TSearch ispell parser code
to client application - probably we would to support not compiled
ispell dictionaries still. Using a precompiler means a new questions
for upgrade!You're inventing a bunch of straw men to attack. There's no reason that
a precompiler approach would have to put any new requirements on the
user. For example, the dictionary-load code could automatically execute
the precompile step if it observed that the precompiled copy of the
dictionary was missing or had an older file timestamp than the source.
uff - just safe activation of precompiler needs lot of low level code
- but maybe I see it wrong, and I doesn't work directly with files
inside pg. But I can't to see it as simple solution.
I like the idea of a precompiler step mainly because it still gives you
most of the benefits of the patch on platforms without mmap. (Instead
of mmap'ing, just open and read() the precompiled file.) In particular,
you would still have a creditable improvement for Windows users without
writing any Windows-specific code.
the loading cca 10 MB takes on my comp cca 30 ms - it is better than
90ms, but it isn't a win.
I think we can divide this problem to three parts
a) simple allocator - it can be used not only for TSearch dictionaries.
I think that's a waste of time, frankly. There aren't enough potential
use cases.b) sharing a data - it is important for large dictionaries
Useful but not really essential.
c) preloading - it decrease load time of first TSearch query
This is the part that is the make-or-break benefit of the patch.
You need a solution that cuts load time even when mmap isn't
available.
I am not sure if this existing, and if it is necessary. Probably main
problem is with Czech language - we have a few specialities. For Czech
environment is UNIX and Windows platform the most important. I have
not information about using Postgres and Fulltext on other platforms
here. So, probably the solution doesn't need be core. I am thinking
about some pgfoundry project now - some like ispell dictionary
preload.
I can send only simplified version without preloading and sharing.
Just solving a memory issue - I think so there are not different
opinions.
best regards
Pavel Stehule
Show quoted text
regards, tom lane
2010/7/8 Tom Lane <tgl@sss.pgh.pa.us>:
For example, the dictionary-load code could automatically execute
the precompile step if it observed that the precompiled copy of the
dictionary was missing or had an older file timestamp than the source.
There might be a problem in automatic precompiler -- Where should we
save the result? OS users of postgres servers don't have write-permission
to $PGSHARE in normal cases. Instead, we can store the precompiled
result to $PGDATA/pg_dict_cache or so.
I like the idea of a precompiler step mainly because it still gives you
most of the benefits of the patch on platforms without mmap.
I also like the precompiler solution. I think the most important benefit
in the approach is that we don't need to declare dictionaries to be preloaded
in configuration files; We can always use mmap() for all dictionary files.
--
Takahiro Itagaki
Itagaki Takahiro <itagaki.takahiro@gmail.com> writes:
2010/7/8 Tom Lane <tgl@sss.pgh.pa.us>:
For example, the dictionary-load code could automatically execute
the precompile step if it observed that the precompiled copy of the
dictionary was missing or had an older file timestamp than the source.
There might be a problem in automatic precompiler -- Where should we
save the result? OS users of postgres servers don't have write-permission
to $PGSHARE in normal cases. Instead, we can store the precompiled
result to $PGDATA/pg_dict_cache or so.
Yeah. Actually we'd *have* to do something like that because $PGSHARE
should contain only architecture-independent files, while the
precompiled files would presumably have dependencies on endianness etc.
regards, tom lane
2010/7/12 Tom Lane <tgl@sss.pgh.pa.us>:
Itagaki Takahiro <itagaki.takahiro@gmail.com> writes:
2010/7/8 Tom Lane <tgl@sss.pgh.pa.us>:
For example, the dictionary-load code could automatically execute
the precompile step if it observed that the precompiled copy of the
dictionary was missing or had an older file timestamp than the source.
I am not sure, but it can be recompiled when tseach code is actualised
(minor update) too.
There might be a problem in automatic precompiler -- Where should we
save the result? OS users of postgres servers don't have write-permission
to $PGSHARE in normal cases. Instead, we can store the precompiled
result to $PGDATA/pg_dict_cache or so.Yeah. Actually we'd *have* to do something like that because $PGSHARE
should contain only architecture-independent files, while the
precompiled files would presumably have dependencies on endianness etc.
It is file and can be broken - so we have to check its consistency.
There have to some evidency of dictionaries in cache - how will be
identified a precompiled file? We cannot use a dictionary name,
because it is a combination of dictionary and stop words. Have to have
to thinking about filenames length here? Will be beter some generated
name and a new system table?
Regards
Pavel Stehule
Show quoted text
regards, tom lane
Hello
this patch is significantly reduced original patch. It doesn't propose
a simple allocator - just eliminate a high memory usage for ispell
dictionary.
without this patch the ispell dictionary takes 55MB for tsearch2
context and 27MB in temp context. With this patch it takes "only" 25MB
tsearch2 context and 19MB in temp context - its for Czech dictionary
and UTF8 encoding. The patch is litlle bit ugly - it was reason, why I
proposed a simple allocator, but it reduce a memory usage on 53% - the
startup is better from 620 to 560 ms ~ 10% faster. little bit strange
is repeated time - it goes down from 18ms to 5ms.
Regards
Pavel Stehule
Attachments:
lessmem.diffapplication/octet-stream; name=lessmem.diffDownload
*** ./src/backend/tsearch/regis.c.orig 2010-07-14 11:47:33.068437888 +0200
--- ./src/backend/tsearch/regis.c 2010-07-14 14:32:47.157437908 +0200
***************
*** 16,27 ****
--- 16,34 ----
#include "tsearch/dicts/regis.h"
#include "tsearch/ts_locale.h"
+ #include "tsearch/dicts/spell.h"
#define RS_IN_ONEOF 1
#define RS_IN_ONEOF_IN 2
#define RS_IN_NONEOF 3
#define RS_IN_WAIT 4
+ extern AllocOnlyBlocksData aob;
+
+ #define aoalloc(sz) ao_alloc(&aob, (sz), false)
+ #define aoalloc0(sz) ao_alloc(&aob, (sz), true)
+ #define aostrdup(str) ao_strdup(&aob, (str))
+
/*
* Test whether a regex is of the subset supported here.
* Keep this in sync with RS_compile!
***************
*** 74,80 ****
{
RegisNode *ptr;
! ptr = (RegisNode *) palloc0(RNHDRSZ + len + 1);
if (prev)
prev->next = ptr;
return ptr;
--- 81,87 ----
{
RegisNode *ptr;
! ptr = (RegisNode *) aoalloc0(RNHDRSZ + len + 1);
if (prev)
prev->next = ptr;
return ptr;
*** ./src/backend/tsearch/spell.c.orig 2010-01-02 17:57:53.000000000 +0100
--- ./src/backend/tsearch/spell.c 2010-07-14 14:46:06.513562368 +0200
***************
*** 28,36 ****
--- 28,97 ----
*/
static MemoryContext tmpCtx = NULL;
+ #define ALLOC_ONLY_MAXSIZE 512
+
+ AllocOnlyBlocksData aob;
+ AllocOnlyBlocksData tmpaob;
+
#define tmpalloc(sz) MemoryContextAlloc(tmpCtx, (sz))
#define tmpalloc0(sz) MemoryContextAllocZero(tmpCtx, (sz))
+ #define aoalloc(sz) ao_alloc(&aob, (sz), false)
+ #define aoalloc0(sz) ao_alloc(&aob, (sz), true)
+ #define aostrdup(str) ao_strdup(&aob, (str))
+
+ #define aotmpalloc(sz) ao_alloc(&tmpaob, (sz), false)
+ #define aotpmalloc0(sz) ao_alloc(&tmpaob, (sz), true)
+ #define aotmpstrdup(str) ao_strdup(&tmpaob, (str))
+
+ /*
+ * Ispell dictionary uses a thousands small blocks and because
+ * doesn't release these blocks, we can save lot of memory via
+ * sharing mcxt blocks.
+ */
+ char *
+ ao_alloc(AllocOnlyBlocksData *aob, Size size, bool fillzero)
+ {
+ char *ptr;
+
+ size = MAXALIGN(size);
+
+ /*
+ * for "larger" blocks use a mcxt allocator. It is simple
+ * mechanism to ensure a well fragmentation of alloc only
+ * blocks.
+ */
+ if (size > ALLOC_ONLY_MAXSIZE)
+ return fillzero ? MemoryContextAllocZero(aob->ctx, size) : MemoryContextAlloc(aob->ctx, size);
+
+ if (size > aob->freesize)
+ {
+ aob->freeptr = MemoryContextAlloc(aob->ctx, ALLOCSET_DEFAULT_INITSIZE);
+ aob->freesize = ALLOCSET_DEFAULT_INITSIZE;
+ }
+
+ ptr = aob->freeptr;
+ aob->freeptr += size;
+ aob->freesize -= size;
+
+ if (fillzero)
+ memset(ptr, 0, size);
+
+ return ptr;
+ }
+
+ char *
+ ao_strdup(AllocOnlyBlocksData *aob, const char *string)
+ {
+ char *nstr;
+ Size len = strlen(string) + 1;
+
+ nstr = (char *) ao_alloc(aob, len, false);
+ memcpy(nstr, string, len);;
+
+ return nstr;
+ }
+
static void
checkTmpCtx(void)
{
***************
*** 48,53 ****
--- 109,120 ----
}
else
tmpCtx = CurrentMemoryContext->firstchild;
+
+ tmpaob.ctx = tmpCtx;
+ tmpaob.freesize = 0;
+
+ aob.ctx = CurrentMemoryContext;
+ aob.freesize = 0;
}
static char *
***************
*** 179,185 ****
Conf->Spell = (SPELL **) tmpalloc(Conf->mspell * sizeof(SPELL *));
}
}
! Conf->Spell[Conf->nspell] = (SPELL *) tmpalloc(SPELLHDRSZ + strlen(word) + 1);
strcpy(Conf->Spell[Conf->nspell]->word, word);
strncpy(Conf->Spell[Conf->nspell]->p.flag, flag, MAXFLAGLEN);
Conf->nspell++;
--- 246,253 ----
Conf->Spell = (SPELL **) tmpalloc(Conf->mspell * sizeof(SPELL *));
}
}
!
! Conf->Spell[Conf->nspell] = (SPELL *) aotmpalloc(SPELLHDRSZ + strlen(word) + 1);
strcpy(Conf->Spell[Conf->nspell]->word, word);
strncpy(Conf->Spell[Conf->nspell]->p.flag, flag, MAXFLAGLEN);
Conf->nspell++;
***************
*** 344,357 ****
Affix->issimple = 0;
Affix->isregis = 0;
! tmask = (char *) tmpalloc(strlen(mask) + 3);
if (type == FF_SUFFIX)
sprintf(tmask, "%s$", mask);
else
sprintf(tmask, "^%s", mask);
masklen = strlen(tmask);
! wmask = (pg_wchar *) tmpalloc((masklen + 1) * sizeof(pg_wchar));
wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen);
err = pg_regcomp(&(Affix->reg.regex), wmask, wmasklen, REG_ADVANCED | REG_NOSUB);
--- 412,425 ----
Affix->issimple = 0;
Affix->isregis = 0;
! tmask = (char *) aotmpalloc(strlen(mask) + 3);
if (type == FF_SUFFIX)
sprintf(tmask, "%s$", mask);
else
sprintf(tmask, "^%s", mask);
masklen = strlen(tmask);
! wmask = (pg_wchar *) aotmpalloc((masklen + 1) * sizeof(pg_wchar));
wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen);
err = pg_regcomp(&(Affix->reg.regex), wmask, wmasklen, REG_ADVANCED | REG_NOSUB);
***************
*** 375,383 ****
Affix->flag = flag;
Affix->type = type;
! Affix->find = (find && *find) ? pstrdup(find) : VoidString;
if ((Affix->replen = strlen(repl)) > 0)
! Affix->repl = pstrdup(repl);
else
Affix->repl = VoidString;
Conf->naffixes++;
--- 443,451 ----
Affix->flag = flag;
Affix->type = type;
! Affix->find = (find && *find) ? aostrdup(find) : VoidString;
if ((Affix->replen = strlen(repl)) > 0)
! Affix->repl = aostrdup(repl);
else
Affix->repl = VoidString;
Conf->naffixes++;
***************
*** 833,839 ****
}
ptr = Conf->AffixData + Conf->nAffixData;
! *ptr = palloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) +
1 /* space */ + 1 /* \0 */ );
sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]);
ptr++;
--- 901,907 ----
}
ptr = Conf->AffixData + Conf->nAffixData;
! *ptr = aoalloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) +
1 /* space */ + 1 /* \0 */ );
sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]);
ptr++;
***************
*** 878,884 ****
if (!nchar)
return NULL;
! rs = (SPNode *) palloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
rs->length = nchar;
data = rs->data;
--- 946,952 ----
if (!nchar)
return NULL;
! rs = (SPNode *) aoalloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
rs->length = nchar;
data = rs->data;
***************
*** 965,971 ****
* dictionary. Replace textual flag-field of Conf->Spell entries with
* indexes into Conf->AffixData array.
*/
! Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
curaffix = -1;
for (i = 0; i < Conf->nspell; i++)
--- 1033,1039 ----
* dictionary. Replace textual flag-field of Conf->Spell entries with
* indexes into Conf->AffixData array.
*/
! Conf->AffixData = (char **) aoalloc0(naffix * sizeof(char *));
curaffix = -1;
for (i = 0; i < Conf->nspell; i++)
***************
*** 974,980 ****
{
curaffix++;
Assert(curaffix < naffix);
! Conf->AffixData[curaffix] = pstrdup(Conf->Spell[i]->p.flag);
}
Conf->Spell[i]->p.d.affix = curaffix;
--- 1042,1048 ----
{
curaffix++;
Assert(curaffix < naffix);
! Conf->AffixData[curaffix] = aostrdup(Conf->Spell[i]->p.flag);
}
Conf->Spell[i]->p.d.affix = curaffix;
***************
*** 1014,1020 ****
aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
naff = 0;
! rs = (AffixNode *) palloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
rs->length = nchar;
data = rs->data;
--- 1082,1088 ----
aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
naff = 0;
! rs = (AffixNode *) aoalloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
rs->length = nchar;
data = rs->data;
***************
*** 1030,1036 ****
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
--- 1098,1104 ----
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) aoalloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
***************
*** 1050,1056 ****
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
--- 1118,1124 ----
if (naff)
{
data->naff = naff;
! data->aff = (AFFIX **) aoalloc(sizeof(AFFIX *) * naff);
memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
naff = 0;
}
***************
*** 1067,1073 ****
cnt = 0;
int start = (issuffix) ? startsuffix : 0;
int end = (issuffix) ? Conf->naffixes : startsuffix;
! AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData));
Affix->length = 1;
Affix->isvoid = 1;
--- 1135,1141 ----
cnt = 0;
int start = (issuffix) ? startsuffix : 0;
int end = (issuffix) ? Conf->naffixes : startsuffix;
! AffixNode *Affix = (AffixNode *) aoalloc0(ANHRDSZ + sizeof(AffixNodeData));
Affix->length = 1;
Affix->isvoid = 1;
***************
*** 1091,1097 ****
if (cnt == 0)
return;
! Affix->data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * cnt);
Affix->data->naff = (uint32) cnt;
cnt = 0;
--- 1159,1165 ----
if (cnt == 0)
return;
! Affix->data->aff = (AFFIX **) aoalloc(sizeof(AFFIX *) * cnt);
Affix->data->naff = (uint32) cnt;
cnt = 0;
***************
*** 1658,1664 ****
addNorm(TSLexeme **lres, TSLexeme **lcur, char *word, int flags, uint16 NVariant)
{
if (*lres == NULL)
! *lcur = *lres = (TSLexeme *) palloc(MAX_NORM * sizeof(TSLexeme));
if (*lcur - *lres < MAX_NORM - 1)
{
--- 1726,1732 ----
addNorm(TSLexeme **lres, TSLexeme **lcur, char *word, int flags, uint16 NVariant)
{
if (*lres == NULL)
! *lcur = *lres = (TSLexeme *) aoalloc(MAX_NORM * sizeof(TSLexeme));
if (*lcur - *lres < MAX_NORM - 1)
{
*** ./src/include/tsearch/dicts/spell.h.orig 2010-04-02 17:21:20.000000000 +0200
--- ./src/include/tsearch/dicts/spell.h 2010-07-14 14:23:42.210440053 +0200
***************
*** 160,169 ****
--- 160,180 ----
bool usecompound;
} IspellDict;
+ typedef struct
+ {
+ MemoryContext ctx;
+ Size freesize;
+ char *freeptr;
+ } AllocOnlyBlocksData;
+
extern TSLexeme *NINormalizeWord(IspellDict *Conf, char *word);
extern void NIImportAffixes(IspellDict *Conf, const char *filename);
extern void NIImportDictionary(IspellDict *Conf, const char *filename);
extern void NISortDictionary(IspellDict *Conf);
extern void NISortAffixes(IspellDict *Conf);
+ extern char *ao_alloc(AllocOnlyBlocksData *aob, Size size, bool fillzero);
+ extern char *ao_strdup(AllocOnlyBlocksData *aob, const char *string);
+
+
#endif
2010/7/14 Pavel Stehule <pavel.stehule@gmail.com>:
this patch is significantly reduced original patch. It doesn't propose
a simple allocator - just eliminate a high memory usage for ispell
dictionary.
I don't think introducing new methods is a good idea. If you want a
simple allocator, MemoryContextMethods layer seems better for me.
The original purpose of the patch -- preloading dictionaries --
is also need to be redesigned with precompiler approach.
I'll move the proposal to "Returned with Feedback" status.
--
Itagaki Takahiro
Hello
2010/7/20 Itagaki Takahiro <itagaki.takahiro@gmail.com>:
2010/7/14 Pavel Stehule <pavel.stehule@gmail.com>:
this patch is significantly reduced original patch. It doesn't propose
a simple allocator - just eliminate a high memory usage for ispell
dictionary.I don't think introducing new methods is a good idea. If you want a
simple allocator, MemoryContextMethods layer seems better for me.
can you explain it, please?
The original purpose of the patch -- preloading dictionaries --
is also need to be redesigned with precompiler approach.
I'll move the proposal to "Returned with Feedback" status.
ok.
thank you very much
Pavel Stehule
Show quoted text
--
Itagaki Takahiro