>From b07280437d5dc3cf60b2755b2562825cd0da0ec6 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Wed, 30 Dec 2015 15:56:56 +0100 Subject: [PATCH 3/5] eliminate the MaxAllocSize limit, now buckets can be up to ~8GB (INT_MAX/2 items) Note: don't forget to zero buckets allocated with AllocHuge() --- src/backend/executor/nodeHash.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index e7d4b5f..c53d485 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -466,22 +466,19 @@ ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew, /* * Set nbuckets to achieve an average bucket load of NTUP_PER_BUCKET when - * memory is filled, assuming a single batch; but limit the value so that - * the pointer arrays we'll try to allocate do not exceed work_mem nor - * MaxAllocSize. + * memory is filled, assuming a single batch. * * Note that both nbuckets and nbatch must be powers of 2 to make * ExecHashGetBucketAndBatch fast. */ max_pointers = (work_mem * 1024L) / sizeof(HashJoinTuple); - max_pointers = Min(max_pointers, MaxAllocSize / sizeof(HashJoinTuple)); + /* If max_pointers isn't a power of 2, must round it down to one */ mppow2 = 1L << my_log2(max_pointers); if (max_pointers != mppow2) max_pointers = mppow2 / 2; - /* Also ensure we avoid integer overflow in nbatch and nbuckets */ - /* (this step is redundant given the current value of MaxAllocSize) */ + /* Ensure we avoid integer overflow in nbatch and nbuckets */ max_pointers = Min(max_pointers, INT_MAX / 2); dbuckets = ceil(ntuples / NTUP_PER_BUCKET); @@ -597,7 +594,7 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) return; /* safety check to avoid overflow */ - if (oldnbatch > Min(INT_MAX / 2, MaxAllocSize / (sizeof(void *) * 2))) + if (oldnbatch > (INT_MAX / 2)) return; /* @@ -757,7 +754,11 @@ ExecHashBuildBuckets(HashJoinTable hashtable) * chunks) */ hashtable->buckets = - (HashJoinTuple *) palloc0(hashtable->nbuckets * sizeof(HashJoinTuple)); + (HashJoinTuple *) MemoryContextAllocHuge(hashtable->batchCxt, + hashtable->nbuckets * sizeof(HashJoinTuple)); + + /* Don't forget to zero the buckets (AllocHuge does not do that). */ + memset(hashtable->buckets, 0, hashtable->nbuckets * sizeof(HashJoinTuple)); /* scan through all tuples in all chunks to rebuild the hash table */ for (chunk = hashtable->chunks; chunk != NULL; chunk = chunk->next) @@ -860,8 +861,7 @@ ExecHashTableInsert(HashJoinTable hashtable, ntuples > (hashtable->nbuckets * NTUP_PER_BUCKET)) { /* Guard against integer overflow and alloc size overflow */ - if (hashtable->nbuckets <= INT_MAX / 2 && - hashtable->nbuckets * 2 <= MaxAllocSize / sizeof(HashJoinTuple)) + if (hashtable->nbuckets <= INT_MAX / 2) { hashtable->nbuckets *= 2; hashtable->log2_nbuckets += 1; @@ -1201,7 +1201,11 @@ ExecHashTableReset(HashJoinTable hashtable) /* Reallocate and reinitialize the hash bucket headers. */ hashtable->buckets = - (HashJoinTuple *) palloc0(hashtable->nbuckets * sizeof(HashJoinTuple)); + (HashJoinTuple *) MemoryContextAllocHuge(hashtable->batchCxt, + hashtable->nbuckets * sizeof(HashJoinTuple)); + + /* Don't forget to zero the buckets (AllocHuge does not do that). */ + memset(hashtable->buckets, 0, hashtable->nbuckets * sizeof(HashJoinTuple)); hashtable->spaceUsed = 0; -- 2.1.0