data-stack.c revision 5517afec377fb7bf80e589b3efdd975f61dad5b3
76b43e4417bab52e913da39b5f5bc2a130d3f149Timo Sirainen/* Copyright (c) 2002-2014 Dovecot authors, see the included COPYING file */
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen/* @UNSAFE: whole file */
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include "lib.h"
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include "data-stack.h"
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#include <stdlib.h>
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainen#ifdef HAVE_GC_GC_H
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen# include <gc/gc.h>
18565c69efcd7db003dbf27cf625ed822e889fb1Timo Sirainen#elif defined (HAVE_GC_H)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen# include <gc.h>
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#endif
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen/* Initial stack size - this should be kept in a size that doesn't exceed
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen in a normal use to avoid extra malloc()ing. */
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#ifdef DEBUG
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen# define INITIAL_STACK_SIZE (1024*10)
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#else
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen# define INITIAL_STACK_SIZE (1024*32)
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#endif
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#ifdef DEBUG
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen# define CLEAR_CHR 0xde
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen# define SENTRY_COUNT (4*8)
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#else
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen# define CLEAR_CHR 0
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#endif
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainenstruct stack_block {
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen struct stack_block *next;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen size_t size, left, lowwater;
af1f4b17a92ca7b2661737e65c7849df289d3070Timo Sirainen /* always NULL and here just in case something accesses
af1f4b17a92ca7b2661737e65c7849df289d3070Timo Sirainen the memory in front of an allocated area */
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen char *nullpad;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen /* unsigned char data[]; */
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen};
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen
01cbf4ac5d44137ab434791be7f838d98d0fcf3bTimo Sirainen#define SIZEOF_MEMBLOCK MEM_ALIGN(sizeof(struct stack_block))
4b41116563110d00330896a568eff1078c382827Timo Sirainen
4b41116563110d00330896a568eff1078c382827Timo Sirainen#define STACK_BLOCK_DATA(block) \
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen ((unsigned char *) (block) + SIZEOF_MEMBLOCK)
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen/* current_frame_block contains last t_push()ed frames. After that new
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen stack_frame_block is created and it's ->prev is set to
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen current_frame_block. */
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen#define BLOCK_FRAME_COUNT 32
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstruct stack_frame_block {
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen struct stack_frame_block *prev;
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen struct stack_block *block[BLOCK_FRAME_COUNT];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen size_t block_space_used[BLOCK_FRAME_COUNT];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen size_t last_alloc_size[BLOCK_FRAME_COUNT];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen#ifdef DEBUG
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen const char *marker[BLOCK_FRAME_COUNT];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen /* Fairly arbitrary profiling data */
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen unsigned long long alloc_bytes[BLOCK_FRAME_COUNT];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen unsigned int alloc_count[BLOCK_FRAME_COUNT];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen#endif
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen};
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen
d30da25fb6be1f1c667d93767c9194000194b618Timo Sirainenunsigned int data_stack_frame = 0;
d30da25fb6be1f1c667d93767c9194000194b618Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic int frame_pos = BLOCK_FRAME_COUNT-1; /* in current_frame_block */
597dce34068d603fb759b4dff404b34049213e51Timo Sirainenstatic struct stack_frame_block *current_frame_block;
63a61b7a739ae0f3f520215137d9c50f94d0f34fTimo Sirainenstatic struct stack_frame_block *unused_frame_blocks;
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic struct stack_block *current_block; /* block now used for allocation */
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic struct stack_block *unused_block; /* largest unused block is kept here */
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic struct stack_block *last_buffer_block;
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic size_t last_buffer_size;
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen#ifdef DEBUG
4b41116563110d00330896a568eff1078c382827Timo Sirainenstatic bool clean_after_pop = TRUE;
910fa4e4204a73d3d24c03f3059dd24e727ca057Timo Sirainen#else
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic bool clean_after_pop = FALSE;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen#endif
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainenstatic bool outofmem = FALSE;
f1901fd21906911f7be075c965ac882f6a87b4c3Timo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic union {
6060b7c8edf8fce73470d0df6a2479b69b01c537Timo Sirainen struct stack_block block;
7f773564b94e6054a40d3785cb63c29f1e4d4deeTimo Sirainen unsigned char data[512];
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen} outofmem_area;
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainenstatic void data_stack_last_buffer_reset(bool preserve_data ATTR_UNUSED)
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen{
fe6c1556d3529a6376d4cbb3766c34aebde0de99Timo Sirainen if (last_buffer_block != NULL) {
4b41116563110d00330896a568eff1078c382827Timo Sirainen#ifdef DEBUG
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen unsigned char *p;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen unsigned int i;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen p = STACK_BLOCK_DATA(current_block) +
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen (current_block->size - current_block->left) +
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen MEM_ALIGN(sizeof(size_t)) + MEM_ALIGN(last_buffer_size);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen#endif
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen /* reset t_buffer_get() mark - not really needed but makes it
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen easier to notice if t_malloc()/t_push()/t_pop() is called
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen between t_buffer_get() and t_buffer_alloc().
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen do this before we get to i_panic() to avoid recursive
4b41116563110d00330896a568eff1078c382827Timo Sirainen panics. */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen last_buffer_block = NULL;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen#ifdef DEBUG
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen for (i = 0; i < SENTRY_COUNT; i++) {
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen if (p[i] != CLEAR_CHR)
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen i_panic("t_buffer_get(): buffer overflow");
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen }
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
345212e8f61ebf14ff4f80df26df9e655eb5121eTimo Sirainen if (!preserve_data) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen p = STACK_BLOCK_DATA(current_block) +
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen (current_block->size - current_block->left);
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen memset(p, CLEAR_CHR, SENTRY_COUNT);
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen#endif
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen }
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen}
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainenunsigned int t_push(const char *marker)
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen{
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct stack_frame_block *frame_block;
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen frame_pos++;
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen if (frame_pos == BLOCK_FRAME_COUNT) {
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen /* frame block full */
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen if (data_stack_frame == 0) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen /* kludgy, but allow this before initialization */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen frame_pos = 0;
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen data_stack_init();
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen return t_push(marker);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen frame_pos = 0;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen if (unused_frame_blocks == NULL) {
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen /* allocate new block */
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#ifndef USE_GC
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen frame_block = calloc(sizeof(*frame_block), 1);
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#else
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen frame_block = GC_malloc(sizeof(*frame_block));
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#endif
4b41116563110d00330896a568eff1078c382827Timo Sirainen if (frame_block == NULL) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen i_fatal_status(FATAL_OUTOFMEM,
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen "t_push(): Out of memory");
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen } else {
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen /* use existing unused frame_block */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen frame_block = unused_frame_blocks;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen unused_frame_blocks = unused_frame_blocks->prev;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen frame_block->prev = current_frame_block;
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen current_frame_block = frame_block;
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen }
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen data_stack_last_buffer_reset(FALSE);
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen /* mark our current position */
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen current_frame_block->block[frame_pos] = current_block;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen current_frame_block->block_space_used[frame_pos] = current_block->left;
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen current_frame_block->last_alloc_size[frame_pos] = 0;
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen#ifdef DEBUG
a0b0d629931773c17a236f6214adbe0e13b9b3fdTimo Sirainen current_frame_block->marker[frame_pos] = marker;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen current_frame_block->alloc_bytes[frame_pos] = 0ULL;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen current_frame_block->alloc_count[frame_pos] = 0;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen#else
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen (void)marker; /* only used for debugging */
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#endif
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen return data_stack_frame++;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen}
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainenunsigned int t_push_named(const char *format, ...)
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen{
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen unsigned int ret = t_push(NULL);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen#ifdef DEBUG
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen va_list args;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen va_start(args, format);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen current_frame_block->marker[frame_pos] = p_strdup_vprintf(unsafe_data_stack_pool, format, args);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen va_end(args);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen#else
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen (void)format; /* unused in non-DEBUG builds */
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen#endif
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen return ret;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen}
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainenstatic void free_blocks(struct stack_block *block)
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen{
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen struct stack_block *next;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen /* free all the blocks, except if any of them is bigger than
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen unused_block, replace it */
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen while (block != NULL) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen next = block->next;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen if (clean_after_pop)
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen memset(STACK_BLOCK_DATA(block), CLEAR_CHR, block->size);
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen if (unused_block == NULL || block->size > unused_block->size) {
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen#ifndef USE_GC
a5c8dc283ef673fcdae158513b8032e74b45f59aTimo Sirainen free(unused_block);
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen#endif
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen unused_block = block;
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen } else {
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen#ifndef USE_GC
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen if (block != &outofmem_area.block)
a5c8dc283ef673fcdae158513b8032e74b45f59aTimo Sirainen free(block);
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen#endif
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
a1808be0774cbcb28fec45341aabf803ec44bae5Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen block = next;
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen}
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen#ifdef DEBUG
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenstatic void t_pop_verify(void)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen{
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen struct stack_block *block;
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen unsigned char *p;
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen size_t pos, max_pos, used_size, alloc_size;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen block = current_frame_block->block[frame_pos];
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen pos = block->size - current_frame_block->block_space_used[frame_pos];
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen while (block != NULL) {
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen used_size = block->size - block->left;
1b5366b2234892f8930a29351da06b193e385150Timo Sirainen p = STACK_BLOCK_DATA(block);
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen while (pos < used_size) {
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen alloc_size = *(size_t *)(p + pos);
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen if (used_size - pos < alloc_size)
aa215ab623706463cea1d047f975ffe51d3f0c05Timo Sirainen i_panic("data stack[%s]: saved alloc size broken",
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen current_frame_block->marker[frame_pos]);
a5c8dc283ef673fcdae158513b8032e74b45f59aTimo Sirainen pos += MEM_ALIGN(sizeof(alloc_size));
a5c8dc283ef673fcdae158513b8032e74b45f59aTimo Sirainen max_pos = pos + MEM_ALIGN(alloc_size + SENTRY_COUNT);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen pos += alloc_size;
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen for (; pos < max_pos; pos++) {
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen if (p[pos] != CLEAR_CHR)
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen i_panic("data stack[%s]: buffer overflow",
b5e6f6f27c1461f0f9f202615eeb738a645188c3Timo Sirainen current_frame_block->marker[frame_pos]);
b2ecd50bb98c44816cb07c17aa17fae2b425f941Timo Sirainen }
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen }
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen /* if we had used t_buffer_get(), the rest of the buffer
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen may not contain CLEAR_CHRs. but we've already checked all
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen the allocations, so there's no need to check them anyway. */
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen block = block->next;
4b41116563110d00330896a568eff1078c382827Timo Sirainen pos = 0;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen}
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen#endif
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainenunsigned int t_pop(void)
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen{
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen struct stack_frame_block *frame_block;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen if (unlikely(frame_pos < 0))
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen i_panic("t_pop() called with empty stack");
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen data_stack_last_buffer_reset(FALSE);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen#ifdef DEBUG
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen t_pop_verify();
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen#endif
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen /* update the current block */
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen current_block = current_frame_block->block[frame_pos];
8f017a40470ef2f4b530000d947a8bce44350a5eTimo Sirainen if (clean_after_pop) {
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen size_t pos, used_size;
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen pos = current_block->size -
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen current_frame_block->block_space_used[frame_pos];
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen used_size = current_block->size - current_block->lowwater;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen memset(STACK_BLOCK_DATA(current_block) + pos, CLEAR_CHR,
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen used_size - pos);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen }
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen current_block->left = current_frame_block->block_space_used[frame_pos];
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen current_block->lowwater = current_block->left;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen if (current_block->next != NULL) {
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen /* free unused blocks */
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen free_blocks(current_block->next);
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen current_block->next = NULL;
64a67c0296b120b6e15169ac9d84dec964e55969Timo Sirainen }
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen if (frame_pos > 0)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen frame_pos--;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen else {
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen /* frame block is now unused, add it to unused list */
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen frame_pos = BLOCK_FRAME_COUNT-1;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen frame_block = current_frame_block;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen current_frame_block = frame_block->prev;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen frame_block->prev = unused_frame_blocks;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen unused_frame_blocks = frame_block;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen }
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen return --data_stack_frame;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen}
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainenvoid t_pop_check(unsigned int *id)
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen{
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen if (unlikely(t_pop() != *id))
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen i_panic("Leaked t_pop() call");
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen *id = 0;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen}
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
0d86aa0d47f7393c669c084b34c0537b193688adTimo Sirainenstatic struct stack_block *mem_block_alloc(size_t min_size)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen{
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen struct stack_block *block;
ae1b268ffff743ad9927c304a1344c5cbd7f909dTimo Sirainen size_t prev_size, alloc_size;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen prev_size = current_block == NULL ? 0 : current_block->size;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen alloc_size = nearest_power(prev_size + min_size);
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen#ifndef USE_GC
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen block = malloc(SIZEOF_MEMBLOCK + alloc_size);
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen#else
90adcaa0a00eba29b7fbd50ca66be11c8d086d6aTimo Sirainen block = GC_malloc(SIZEOF_MEMBLOCK + alloc_size);
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen#endif
d67fde1a8ebc1d85704c5986d8f93aae97eccef3Timo Sirainen if (unlikely(block == NULL)) {
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen if (outofmem) {
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen if (min_size > outofmem_area.block.left)
87460b08cb97b31cde640d4975a6aa2c1d0e7226Timo Sirainen abort();
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen return &outofmem_area.block;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen }
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen outofmem = TRUE;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen i_panic("data stack: Out of memory when allocating %"
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen PRIuSIZE_T" bytes", alloc_size + SIZEOF_MEMBLOCK);
01404d41657a104c5ea1c12bb87f9c321e9c1ac4Timo Sirainen }
01404d41657a104c5ea1c12bb87f9c321e9c1ac4Timo Sirainen block->size = alloc_size;
8fcff4c5b52f24d9c681805fdf06b486f1d0fcbeTimo Sirainen block->left = 0;
block->lowwater = block->size;
block->next = NULL;
block->nullpad = NULL;
#ifdef DEBUG
memset(STACK_BLOCK_DATA(block), CLEAR_CHR, alloc_size);
#endif
return block;
}
static void *t_malloc_real(size_t size, bool permanent)
{
struct stack_block *block;
void *ret;
size_t alloc_size;
#ifdef DEBUG
bool warn = FALSE;
int old_errno = errno;
#endif
if (unlikely(size == 0 || size > SSIZE_T_MAX))
i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
if (unlikely(data_stack_frame == 0)) {
/* kludgy, but allow this before initialization */
data_stack_init();
}
/* allocate only aligned amount of memory so alignment comes
always properly */
#ifndef DEBUG
alloc_size = MEM_ALIGN(size);
#else
alloc_size = MEM_ALIGN(sizeof(size)) + MEM_ALIGN(size + SENTRY_COUNT);
if(permanent) {
current_frame_block->alloc_bytes[frame_pos] += alloc_size;
current_frame_block->alloc_count[frame_pos]++;
}
#endif
data_stack_last_buffer_reset(TRUE);
/* used for t_try_realloc() */
current_frame_block->last_alloc_size[frame_pos] = alloc_size;
if (current_block->left >= alloc_size) {
/* enough space in current block, use it */
ret = STACK_BLOCK_DATA(current_block) +
(current_block->size - current_block->left);
if (current_block->left - alloc_size <
current_block->lowwater) {
current_block->lowwater =
current_block->left - alloc_size;
}
if (permanent)
current_block->left -= alloc_size;
} else {
/* current block is full, see if we can use the unused_block */
if (unused_block != NULL && unused_block->size >= alloc_size) {
block = unused_block;
unused_block = NULL;
} else {
block = mem_block_alloc(alloc_size);
#ifdef DEBUG
warn = TRUE;
#endif
}
block->left = block->size;
if (block->left - alloc_size < block->lowwater)
block->lowwater = block->left - alloc_size;
if (permanent)
block->left -= alloc_size;
block->next = NULL;
current_block->next = block;
current_block = block;
ret = STACK_BLOCK_DATA(current_block);
#ifdef DEBUG
if (warn && getenv("DEBUG_SILENT") == NULL) {
/* warn after allocation, so if i_warning() wants to
allocate more memory we don't go to infinite loop */
i_warning("Growing data stack by %"PRIuSIZE_T" as "
"'%s' reaches %llu bytes from %u allocations.",
block->size,
current_frame_block->marker[frame_pos],
current_frame_block->alloc_bytes[frame_pos],
current_frame_block->alloc_count[frame_pos]);
}
#endif
}
#ifdef DEBUG
memcpy(ret, &size, sizeof(size));
ret = PTR_OFFSET(ret, MEM_ALIGN(sizeof(size)));
/* make sure the sentry contains CLEAR_CHRs. it might not if we
had used t_buffer_get(). */
memset(PTR_OFFSET(ret, size), CLEAR_CHR,
MEM_ALIGN(size + SENTRY_COUNT) - size);
/* we rely on errno not changing. it shouldn't. */
i_assert(errno == old_errno);
#endif
return ret;
}
void *t_malloc(size_t size)
{
return t_malloc_real(size, TRUE);
}
void *t_malloc0(size_t size)
{
void *mem;
mem = t_malloc_real(size, TRUE);
memset(mem, 0, size);
return mem;
}
bool t_try_realloc(void *mem, size_t size)
{
size_t last_alloc_size;
if (unlikely(size == 0 || size > SSIZE_T_MAX))
i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
last_alloc_size = current_frame_block->last_alloc_size[frame_pos];
/* see if we're trying to grow the memory we allocated last */
if (STACK_BLOCK_DATA(current_block) +
(current_block->size - current_block->left -
last_alloc_size) == mem) {
/* yeah, see if we have space to grow */
size = MEM_ALIGN(size);
if (current_block->left >= size - last_alloc_size) {
/* just shrink the available size */
current_block->left -= size - last_alloc_size;
current_frame_block->last_alloc_size[frame_pos] = size;
return TRUE;
}
}
return FALSE;
}
size_t t_get_bytes_available(void)
{
#ifndef DEBUG
const unsigned int extra = MEM_ALIGN_SIZE-1;
#else
const unsigned int extra = MEM_ALIGN_SIZE-1 + SENTRY_COUNT +
MEM_ALIGN(sizeof(size_t));
#endif
return current_block->left < extra ? current_block->left :
current_block->left - extra;
}
void *t_buffer_get(size_t size)
{
void *ret;
ret = t_malloc_real(size, FALSE);
last_buffer_size = size;
last_buffer_block = current_block;
return ret;
}
void *t_buffer_reget(void *buffer, size_t size)
{
size_t old_size;
void *new_buffer;
old_size = last_buffer_size;
if (size <= old_size)
return buffer;
new_buffer = t_buffer_get(size);
if (new_buffer != buffer)
memcpy(new_buffer, buffer, old_size);
return new_buffer;
}
void t_buffer_alloc(size_t size)
{
i_assert(last_buffer_block != NULL);
i_assert(last_buffer_size >= size);
i_assert(current_block->left >= size);
/* we've already reserved the space, now we just mark it used */
(void)t_malloc_real(size, TRUE);
}
void t_buffer_alloc_last_full(void)
{
if (last_buffer_block != NULL)
(void)t_malloc_real(last_buffer_size, TRUE);
}
void data_stack_set_clean_after_pop(bool enable ATTR_UNUSED)
{
#ifndef DEBUG
clean_after_pop = enable;
#endif
}
void data_stack_init(void)
{
if (data_stack_frame > 0) {
/* already initialized (we did auto-initialization in
t_malloc/t_push) */
return;
}
data_stack_frame = 1;
outofmem_area.block.size = outofmem_area.block.left =
sizeof(outofmem_area) - sizeof(outofmem_area.block);
current_block = mem_block_alloc(INITIAL_STACK_SIZE);
current_block->left = current_block->size;
current_block->next = NULL;
current_frame_block = NULL;
unused_frame_blocks = NULL;
frame_pos = BLOCK_FRAME_COUNT-1;
last_buffer_block = NULL;
last_buffer_size = 0;
(void)t_push("data_stack_init");
}
void data_stack_deinit(void)
{
(void)t_pop();
if (frame_pos != BLOCK_FRAME_COUNT-1)
i_panic("Missing t_pop() call");
#ifndef USE_GC
while (unused_frame_blocks != NULL) {
struct stack_frame_block *frame_block = unused_frame_blocks;
unused_frame_blocks = unused_frame_blocks->prev;
free(frame_block);
}
free(current_block);
free(unused_block);
#endif
unused_frame_blocks = NULL;
current_block = NULL;
unused_block = NULL;
}