data-stack.c revision d9d576a2fd9683d2bbd758868c30be75654a6166
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen/* Copyright (c) 2002-2014 Dovecot authors, see the included COPYING file */
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen/* @UNSAFE: whole file */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen/* Initial stack size - this should be kept in a size that doesn't exceed
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen in a normal use to avoid extra malloc()ing. */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen# define CLEAR_CHR 0xD5 /* D5 is mnemonic for "Data 5tack" */
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen# define BLOCK_CANARY ((void *)0xBADBADD5BADBADD5) /* contains 'D5' */
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen# define BLOCK_CANARY_CHECK(block) i_assert((block)->canary == BLOCK_CANARY)
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen# define BLOCK_CANARY_CHECK(block) do { ; } while(0)
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen /* NULL or a poison value, just in case something accesses
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen the memory in front of an allocated area */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* unsigned char data[]; */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen#define SIZEOF_MEMBLOCK MEM_ALIGN(sizeof(struct stack_block))
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen/* current_frame_block contains last t_push()ed frames. After that new
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen stack_frame_block is created and it's ->prev is set to
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen current_frame_block. */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen /* Fairly arbitrary profiling data */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen unsigned long long alloc_bytes[BLOCK_FRAME_COUNT];
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainenunsigned int data_stack_frame = 0;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainenstatic int frame_pos = BLOCK_FRAME_COUNT-1; /* in current_frame_block */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainenstatic struct stack_frame_block *current_frame_block;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainenstatic struct stack_frame_block *unused_frame_blocks;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainenstatic struct stack_block *current_block; /* block now used for allocation */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainenstatic struct stack_block *unused_block; /* largest unused block is kept here */
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainenstatic union {
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainenstatic void data_stack_last_buffer_reset(bool preserve_data ATTR_UNUSED)
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen unsigned char *last_alloc_end, *p;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen unsigned int i;
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen last_alloc_end = STACK_BLOCK_DATA(current_block) +
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen p = last_alloc_end + MEM_ALIGN(sizeof(size_t)) + MEM_ALIGN(last_buffer_size);
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen /* reset t_buffer_get() mark - not really needed but makes it
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen easier to notice if t_malloc()/t_push()/t_pop() is called
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen between t_buffer_get() and t_buffer_alloc().
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen do this before we get to i_panic() to avoid recursive
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen for (i = 0; i < SENTRY_COUNT; i++) {
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* frame block full */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* kludgy, but allow this before initialization */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* allocate new block */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen frame_block = calloc(sizeof(*frame_block), 1);
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen frame_block = GC_malloc(sizeof(*frame_block));
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen "t_push(): Out of memory");
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* use existing unused frame_block */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen unused_frame_blocks = unused_frame_blocks->prev;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* mark our current position */
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->block[frame_pos] = current_block;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->block_space_used[frame_pos] = current_block->left;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->last_alloc_size[frame_pos] = 0;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->marker[frame_pos] = marker;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->alloc_bytes[frame_pos] = 0ULL;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->alloc_count[frame_pos] = 0;
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainenunsigned int t_push_named(const char *format, ...)
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen current_frame_block->marker[frame_pos] = p_strdup_vprintf(unsafe_data_stack_pool, format, args);
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen (void)format; /* unused in non-DEBUG builds */
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainenstatic void free_blocks(struct stack_block *block)
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen /* free all the blocks, except if any of them is bigger than
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen unused_block, replace it */
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen memset(STACK_BLOCK_DATA(block), CLEAR_CHR, block->size);
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen if (unused_block == NULL || block->size > unused_block->size) {
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainenstatic void t_pop_verify(void)
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen unsigned char *p;
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen block = current_frame_block->block[frame_pos];
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen pos = block->size - current_frame_block->block_space_used[frame_pos];
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen i_panic("data stack[%s]: saved alloc size broken",
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen max_pos = pos + MEM_ALIGN(requested_size + SENTRY_COUNT);
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen /* if we had used t_buffer_get(), the rest of the buffer
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen may not contain CLEAR_CHRs. but we've already checked all
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen the allocations, so there's no need to check them anyway. */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainenunsigned int t_pop(void)
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen /* update the current block */
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen current_block = current_frame_block->block[frame_pos];
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen current_frame_block->block_space_used[frame_pos];
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen used_size = current_block->size - current_block->lowwater;
632018810af689442569cbb0139c55868923ccfeTimo Sirainen memset(STACK_BLOCK_DATA(current_block) + pos, CLEAR_CHR,
632018810af689442569cbb0139c55868923ccfeTimo Sirainen current_block->left = current_frame_block->block_space_used[frame_pos];
632018810af689442569cbb0139c55868923ccfeTimo Sirainen current_block->lowwater = current_block->left;
632018810af689442569cbb0139c55868923ccfeTimo Sirainen /* free unused blocks */
071543cc13df9600d2e97aa35f28907be5a79477Timo Sirainen /* frame block is now unused, add it to unused list */
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainenstatic struct stack_block *mem_block_alloc(size_t min_size)
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen prev_size = current_block == NULL ? 0 : current_block->size;
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen alloc_size = nearest_power(prev_size + min_size);
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen block = GC_malloc(SIZEOF_MEMBLOCK + alloc_size);
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen i_panic("data stack: Out of memory when allocating %"
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen PRIuSIZE_T" bytes", alloc_size + SIZEOF_MEMBLOCK);
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen memset(STACK_BLOCK_DATA(block), CLEAR_CHR, alloc_size);
8d3af185ae454653fad60e41c5f36edb1d45c868Timo Sirainenstatic void *t_malloc_real(size_t size, bool permanent)
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen if (unlikely(size == 0 || size > SSIZE_T_MAX))
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen /* kludgy, but allow this before initialization */
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen /* allocate only aligned amount of memory so alignment comes
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen always properly */
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen alloc_size = MEM_ALIGN(sizeof(size)) + MEM_ALIGN(size + SENTRY_COUNT);
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen current_frame_block->alloc_bytes[frame_pos] += alloc_size;
27e859cee42654bff801ba96677cfc4e4e0108c7Timo Sirainen current_frame_block->alloc_count[frame_pos]++;
dea8bfa31729dbdde1b12718f1ef98fac4e99db9Timo Sirainen /* used for t_try_realloc() */
632018810af689442569cbb0139c55868923ccfeTimo Sirainen current_frame_block->last_alloc_size[frame_pos] = alloc_size;
071543cc13df9600d2e97aa35f28907be5a79477Timo Sirainen /* enough space in current block, use it */
632018810af689442569cbb0139c55868923ccfeTimo Sirainen /* current block is full, see if we can use the unused_block */
071543cc13df9600d2e97aa35f28907be5a79477Timo Sirainen if (unused_block != NULL && unused_block->size >= alloc_size) {
071543cc13df9600d2e97aa35f28907be5a79477Timo Sirainen if (block->left - alloc_size < block->lowwater)
632018810af689442569cbb0139c55868923ccfeTimo Sirainen /* warn after allocation, so if i_warning() wants to
632018810af689442569cbb0139c55868923ccfeTimo Sirainen allocate more memory we don't go to infinite loop */
dea8bfa31729dbdde1b12718f1ef98fac4e99db9Timo Sirainen i_warning("Growing data stack by %"PRIuSIZE_T" as "
632018810af689442569cbb0139c55868923ccfeTimo Sirainen "'%s' reaches %llu bytes from %u allocations.",
071543cc13df9600d2e97aa35f28907be5a79477Timo Sirainen ret = PTR_OFFSET(ret, MEM_ALIGN(sizeof(size)));
632018810af689442569cbb0139c55868923ccfeTimo Sirainen /* make sure the sentry contains CLEAR_CHRs. it might not if we
632018810af689442569cbb0139c55868923ccfeTimo Sirainen had used t_buffer_get(). */
632018810af689442569cbb0139c55868923ccfeTimo Sirainen /* we rely on errno not changing. it shouldn't. */
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen if (unlikely(size == 0 || size > SSIZE_T_MAX))
7000810786f2959f02cd6d2f4151a9eb61ff5db8Timo Sirainen i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen last_alloc_size = current_frame_block->last_alloc_size[frame_pos];
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* see if we're trying to grow the memory we allocated last */
cc935aff970ed6c24d136cc560c7e705a49d536cTimo Sirainen /* yeah, see if we have space to grow */
37ce8d49a351f073958624b78c702af75362c1cbTimo Sirainen if (current_block->left >= size - last_alloc_size) {
37ce8d49a351f073958624b78c702af75362c1cbTimo Sirainen /* just shrink the available size */
37ce8d49a351f073958624b78c702af75362c1cbTimo Sirainen current_block->left -= size - last_alloc_size;
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen current_frame_block->last_alloc_size[frame_pos] = size;
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen const unsigned int extra = MEM_ALIGN_SIZE-1 + SENTRY_COUNT +
f8740ac53310cd28ba4ec6dc9e9ce6e9a3688f39Timo Sirainen return current_block->left < extra ? current_block->left :
071543cc13df9600d2e97aa35f28907be5a79477Timo Sirainenvoid *t_buffer_reget(void *buffer, size_t size)
c9099b08b3cae8a849098ca776b4363c6d5f5f36Timo Sirainen /* we've already reserved the space, now we just mark it used */
c9099b08b3cae8a849098ca776b4363c6d5f5f36Timo Sirainenvoid data_stack_set_clean_after_pop(bool enable ATTR_UNUSED)
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen /* already initialized (we did auto-initialization in
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen outofmem_area.block.size = outofmem_area.block.left =
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen sizeof(outofmem_area) - sizeof(outofmem_area.block);
f5e8a76a128d4e92f0641135183c164fd5c5ce5eTimo Sirainen current_block = mem_block_alloc(INITIAL_STACK_SIZE);
d6a7cb184cc882a90aa3d9312082e0029f354ff6Timo Sirainen struct stack_frame_block *frame_block = unused_frame_blocks;