Adds accounting of memory allocated in a memory context. Compared to
various ad hoc solutions, the main advantage is that the accounting is
transparent and does not require direct control over allocations (this
matters for use cases where the allocations happen in user code, like
for example aggregate states allocated in a transition functions).
To reduce overhead, the accounting happens at the block level (not for
individual chunks) and only the context immediately owning the block is
updated. When inquiring about amount of memory allocated in a context,
we have to recursively walk all children contexts.
This "lazy" accounting works well for cases with relatively small number
of contexts in the relevant subtree and/or with infrequent inquiries.
Author: Jeff Davis
Reivewed-by: Tomas Vondra, Melanie Plageman, Soumyadeep Chakraborty
Discussion: https://www.postgresql.org/message-id/flat/
027a129b8525601c6a680d27ce3a7172dab61aab.camel@j-davis.com
* reset a context (free all memory allocated in the context, but not the
context object itself)
+* inquire about the total amount of memory allocated to the context
+ (the raw memory from which the context allocates chunks; not the
+ chunks themselves)
+
Given a chunk of memory previously allocated from a context, one can
free it or reallocate it larger or smaller (corresponding to standard C
library's free() and realloc() routines). These operations return memory
These memory contexts were initially developed for ReorderBuffer, but
may be useful elsewhere as long as the allocation patterns match.
+
+
+Memory Accounting
+-----------------
+
+One of the basic memory context operations is determining the amount of
+memory used in the context (and it's children). We have multiple places
+that implement their own ad hoc memory accounting, and this is meant to
+provide a unified approach. Ad hoc accounting solutions work for places
+with tight control over the allocations or when it's easy to determine
+sizes of allocated chunks (e.g. places that only work with tuples).
+
+The accounting built into the memory contexts is transparent and works
+transparently for all allocations as long as they end up in the right
+memory context subtree.
+
+Consider for example aggregate functions - the aggregate state is often
+represented by an arbitrary structure, allocated from the transition
+function, so the ad hoc accounting is unlikely to work. The built-in
+accounting will however handle such cases just fine.
+
+To minimize overhead, the accounting is done at the block level, not for
+individual allocation chunks.
+
+The accounting is lazy - after a block is allocated (or freed), only the
+context owning that block is updated. This means that when inquiring
+about the memory usage in a given context, we have to walk all children
+contexts recursively. This means the memory accounting is not intended
+for cases with too many memory contexts (in the relevant subtree).
+
parent,
name);
+ ((MemoryContext) set)->mem_allocated =
+ set->keeper->endptr - ((char *) set);
+
return (MemoryContext) set;
}
}
parent,
name);
+ ((MemoryContext) set)->mem_allocated = firstBlockSize;
+
return (MemoryContext) set;
}
{
AllocSet set = (AllocSet) context;
AllocBlock block;
+ Size keepersize = set->keeper->endptr - ((char *) set);
AssertArg(AllocSetIsValid(set));
else
{
/* Normal case, release the block */
+ context->mem_allocated -= block->endptr - ((char*) block);
+
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
#endif
block = next;
}
+ Assert(context->mem_allocated == keepersize);
+
/* Reset block size allocation sequence, too */
set->nextBlockSize = set->initBlockSize;
}
{
AllocSet set = (AllocSet) context;
AllocBlock block = set->blocks;
+ Size keepersize = set->keeper->endptr - ((char *) set);
AssertArg(AllocSetIsValid(set));
{
AllocBlock next = block->next;
+ if (block != set->keeper)
+ context->mem_allocated -= block->endptr - ((char *) block);
+
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
#endif
block = next;
}
+ Assert(context->mem_allocated == keepersize);
+
/* Finally, free the context header, including the keeper block */
free(set);
}
block = (AllocBlock) malloc(blksize);
if (block == NULL)
return NULL;
+
+ context->mem_allocated += blksize;
+
block->aset = set;
block->freeptr = block->endptr = ((char *) block) + blksize;
if (block == NULL)
return NULL;
+ context->mem_allocated += blksize;
+
block->aset = set;
block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
block->endptr = ((char *) block) + blksize;
set->blocks = block->next;
if (block->next)
block->next->prev = block->prev;
+
+ context->mem_allocated -= block->endptr - ((char*) block);
+
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
#endif
AllocBlock block = (AllocBlock) (((char *) chunk) - ALLOC_BLOCKHDRSZ);
Size chksize;
Size blksize;
+ Size oldblksize;
/*
* Try to verify that we have a sane block pointer: it should
/* Do the realloc */
chksize = MAXALIGN(size);
blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
+ oldblksize = block->endptr - ((char *)block);
+
block = (AllocBlock) realloc(block, blksize);
if (block == NULL)
{
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOCCHUNK_PRIVATE_LEN);
return NULL;
}
+
+ context->mem_allocated += blksize - oldblksize;
+
block->freeptr = block->endptr = ((char *) block) + blksize;
/* Update pointers since block has likely been moved */
const char *name = set->header.name;
AllocBlock prevblock;
AllocBlock block;
+ int64 total_allocated = 0;
for (prevblock = NULL, block = set->blocks;
block != NULL;
long blk_data = 0;
long nchunks = 0;
+ if (set->keeper == block)
+ total_allocated += block->endptr - ((char *) set);
+ else
+ total_allocated += block->endptr - ((char *) block);
+
/*
* Empty block - empty can be keeper-block only
*/
elog(WARNING, "problem in alloc set %s: found inconsistent memory block %p",
name, block);
}
+
+ Assert(total_allocated == context->mem_allocated);
}
#endif /* MEMORY_CONTEXT_CHECKING */
dlist_delete(miter.cur);
+ context->mem_allocated -= block->blksize;
+
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->blksize);
#endif
if (block == NULL)
return NULL;
+ context->mem_allocated += blksize;
+
/* block with a single (used) chunk */
block->blksize = blksize;
block->nchunks = 1;
if (block == NULL)
return NULL;
+ context->mem_allocated += blksize;
+
block->blksize = blksize;
block->nchunks = 0;
block->nfree = 0;
if (set->block == block)
set->block = NULL;
+ context->mem_allocated -= block->blksize;
free(block);
}
GenerationContext *gen = (GenerationContext *) context;
const char *name = context->name;
dlist_iter iter;
+ int64 total_allocated = 0;
/* walk all blocks in this context */
dlist_foreach(iter, &gen->blocks)
nchunks;
char *ptr;
+ total_allocated += block->blksize;
+
/*
* nfree > nchunks is surely wrong, and we don't expect to see
* equality either, because such a block should have gotten freed.
elog(WARNING, "problem in Generation %s: number of free chunks %d in block %p does not match header %d",
name, nfree, block, block->nfree);
}
+
+ Assert(total_allocated == context->mem_allocated);
}
#endif /* MEMORY_CONTEXT_CHECKING */
return context->methods->is_empty(context);
}
+/*
+ * Find the memory allocated to blocks for this memory context. If recurse is
+ * true, also include children.
+ */
+int64
+MemoryContextMemAllocated(MemoryContext context, bool recurse)
+{
+ int64 total = context->mem_allocated;
+
+ AssertArg(MemoryContextIsValid(context));
+
+ if (recurse)
+ {
+ MemoryContext child = context->firstchild;
+
+ for (child = context->firstchild;
+ child != NULL;
+ child = child->nextchild)
+ total += MemoryContextMemAllocated(child, true);
+ }
+
+ return total;
+}
+
/*
* MemoryContextStats
* Print statistics about the named context and all its descendants.
node->methods = methods;
node->parent = parent;
node->firstchild = NULL;
+ node->mem_allocated = 0;
node->prevchild = NULL;
node->name = name;
node->ident = NULL;
#endif
free(block);
slab->nblocks--;
+ context->mem_allocated -= slab->blockSize;
}
}
slab->minFreeChunks = 0;
Assert(slab->nblocks == 0);
+ Assert(context->mem_allocated == 0);
}
/*
slab->minFreeChunks&nb
projects / postgresql-pgindent.git / commitdiff