2ronwalf

2ronwalf#include "lib.h"

2ronwalf#include "array.h"

2ronwalf#include "buffer.h"

2ronwalf#include "str.h"

2ronwalf#include "mail-cache-private.h"

2ronwalf

2ronwalf#include <stdlib.h>

2ronwalf

2ronwalf#define CACHE_PREFETCH IO_BLOCK_SIZE

2ronwalf

2ronwalfint mail_cache_get_record(struct mail_cache *cache, uint32_t offset,

2ronwalf const struct mail_cache_record **rec_r)

2ronwalf{

2ronwalf const struct mail_cache_record *rec;

2ronwalf

2ronwalf i_assert(offset != 0);

2ronwalf

2ronwalf if (offset % sizeof(uint32_t) != 0) {

2ronwalf /* records are always 32-bit aligned */

2ronwalf mail_cache_set_corrupted(cache, "invalid record offset");

2ronwalf return -1;

2ronwalf }

2ronwalf

2ronwalf /* we don't know yet how large the record is, so just guess */

2ronwalf if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH) < 0)

2ronwalf return -1;

2ronwalf

2ronwalf if (offset + sizeof(*rec) > cache->mmap_length) {

2ronwalf mail_cache_set_corrupted(cache, "record points outside file");

2ronwalf return -1;

2ronwalf }

2ronwalf rec = CACHE_RECORD(cache, offset);

2ronwalf

2ronwalf if (rec->size < sizeof(*rec)) {

2ronwalf mail_cache_set_corrupted(cache, "invalid record size");

2ronwalf return -1;

2ronwalf }

2ronwalf if (rec->size > CACHE_PREFETCH) {

2ronwalf /* larger than we guessed. map the rest of the record. */

2ronwalf if (mail_cache_map(cache, offset, rec->size) < 0)

2ronwalf return -1;

2ronwalf rec = CACHE_RECORD(cache, offset);

2ronwalf }

2ronwalf

2ronwalf if (rec->size > cache->mmap_length ||

2ronwalf offset + rec->size > cache->mmap_length) {

2ronwalf mail_cache_set_corrupted(cache, "record points outside file");

2ronwalf return -1;

2ronwalf }

2ronwalf

2ronwalf *rec_r = rec;

2ronwalf return 0;

2ronwalf}

2ronwalf

2ronwalfuint32_t mail_cache_lookup_cur_offset(struct mail_index_view *view,

2ronwalf uint32_t seq, uint32_t *reset_id_r)

2ronwalf{

2ronwalf struct mail_cache *cache = mail_index_view_get_index(view)->cache;

2ronwalf struct mail_index_map *map;

2ronwalf const void *data;

2ronwalf uint32_t offset;

2ronwalf

2ronwalf mail_index_lookup_ext_full(view, seq, cache->ext_id,

2ronwalf &map, &data, NULL);

2ronwalf if (data == NULL) {

2ronwalf /* no cache offsets */

2ronwalf return 0;

2ronwalf }

2ronwalf offset = *((const uint32_t *)data);

2ronwalf if (offset == 0)

2ronwalf return 0;

2ronwalf

2ronwalf if (!mail_index_ext_get_reset_id(view, map, cache->ext_id, reset_id_r))

2ronwalf i_unreached();

2ronwalf return offset;

2ronwalf}

2ronwalf

2ronwalfstatic int

2ronwalfmail_cache_lookup_offset(struct mail_cache *cache, struct mail_index_view *view,

2ronwalf uint32_t seq, uint32_t *offset_r)

2ronwalf{

2ronwalf uint32_t offset, reset_id;

2ronwalf int i, ret;

2ronwalf

2ronwalf offset = mail_cache_lookup_cur_offset(view, seq, &reset_id);

2ronwalf if (offset == 0)

2ronwalf return 0;

2ronwalf

2ronwalf /* reset_id must match file_seq or the offset is for a different cache

2ronwalf i = 0;

2ronwalf while (cache->hdr->file_seq != reset_id) {

2ronwalf if (++i == 2 || reset_id < cache->hdr->file_seq)

2ronwalf return 0;

2ronwalf

2ronwalf if (cache->locked) {

2ronwalf /* we're probably compressing */

2ronwalf return 0;

2ronwalf }

2ronwalf

2ronwalf if ((ret = mail_cache_reopen(cache)) <= 0) {

2ronwalf /* error / we already have the latest file open */

2ronwalf return ret;

2ronwalf }

2ronwalf }

2ronwalf

2ronwalf *offset_r = offset;

2ronwalf return 1;

2ronwalf}

2ronwalf

2ronwalfbool mail_cache_track_loops(struct mail_cache_loop_track *loop_track,

2ronwalf uoff_t offset, uoff_t size)

2ronwalf{

2ronwalf i_assert(offset != 0);

2ronwalf i_assert(size != 0);

2ronwalf

2ronwalf /* looping happens only in rare error conditions, so it's enough if we

2ronwalf if (loop_track->size_sum == 0) {

2ronwalf /* first call */

2ronwalf loop_track->min_offset = offset;

2ronwalf loop_track->max_offset = offset + size;

2ronwalf } else {

2ronwalf if (loop_track->min_offset > offset)

2ronwalf loop_track->min_offset = offset;

2ronwalf if (loop_track->max_offset < offset + size)

2ronwalf loop_track->max_offset = offset + size;

2ronwalf }

2ronwalf

2ronwalf loop_track->size_sum += size;

2ronwalf return loop_track->size_sum >

2ronwalf (loop_track->max_offset - loop_track->min_offset);

2ronwalf}

2ronwalf

2ronwalfvoid mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq,

2ronwalf struct mail_cache_lookup_iterate_ctx *ctx_r)

2ronwalf{

2ronwalf struct mail_cache_lookup_iterate_ctx *ctx = ctx_r;

2ronwalf int ret;

2ronwalf

2ronwalf if (!view->cache->opened)

2ronwalf (void)mail_cache_open_and_verify(view->cache);

2ronwalf

2ronwalf memset(ctx, 0, sizeof(*ctx));

2ronwalf ctx->view = view;

2ronwalf ctx->seq = seq;

2ronwalf

2ronwalf if (!MAIL_CACHE_IS_UNUSABLE(view->cache)) {

2ronwalf /* look up the first offset */

2ronwalf ret = mail_cache_lookup_offset(view->cache, view->view, seq,

2ronwalf &ctx->offset);

2ronwalf if (ret <= 0) {

2ronwalf ctx->stop = TRUE;

2ronwalf ctx->failed = ret < 0;

2ronwalf }

2ronwalf }

2ronwalf ctx->remap_counter = view->cache->remap_counter;

2ronwalf

2ronwalf memset(&view->loop_track, 0, sizeof(view->loop_track));

2ronwalf}

2ronwalf

2ronwalfstatic int

2ronwalfmail_cache_lookup_iter_next_record(struct mail_cache_lookup_iterate_ctx *ctx)

2ronwalf{

2ronwalf struct mail_cache_view *view = ctx->view;

2ronwalf

2ronwalf if (ctx->stop)

2ronwalf return ctx->failed ? -1 : 0;

2ronwalf

2ronwalf if (ctx->rec != NULL)

2ronwalf ctx->offset = ctx->rec->prev_offset;

2ronwalf if (ctx->offset == 0) {

2ronwalf /* end of this record list. check newly appended data. */

2ronwalf if (ctx->appends_checked ||

2ronwalf view->trans_seq1 > ctx->seq ||

2ronwalf view->trans_seq2 < ctx->seq ||

2ronwalf MAIL_CACHE_IS_UNUSABLE(view->cache) ||

2ronwalf mail_cache_lookup_offset(view->cache, view->trans_view,

2ronwalf ctx->seq, &ctx->offset) <= 0)

2ronwalf return 0;

2ronwalf

2ronwalf ctx->appends_checked = TRUE;

2ronwalf ctx->remap_counter = view->cache->remap_counter;

2ronwalf memset(&view->loop_track, 0, sizeof(view->loop_track));

2ronwalf }

2ronwalf

2ronwalf /* look up the next record */

2ronwalf if (mail_cache_get_record(view->cache, ctx->offset, &ctx->rec) < 0)

2ronwalf return -1;

2ronwalf if (mail_cache_track_loops(&view->loop_track, ctx->offset,

2ronwalf ctx->rec->size)) {

2ronwalf mail_cache_set_corrupted(view->cache,

2ronwalf "record list is circular");

2ronwalf return -1;

2ronwalf }

ctx->remap_counter = view->cache->remap_counter;

ctx->pos = sizeof(*ctx->rec);

ctx->rec_size = ctx->rec->size;

return 1;

}

int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx,

struct mail_cache_iterate_field *field_r)

{

struct mail_cache *cache = ctx->view->cache;

unsigned int field_idx;

unsigned int data_size;

uint32_t file_field;

int ret;

i_assert(ctx->remap_counter == cache->remap_counter);

if (ctx->pos + sizeof(uint32_t) > ctx->rec_size) {

if (ctx->pos != ctx->rec_size) {

mail_cache_set_corrupted(cache,

"record has invalid size");

return -1;

}

if ((ret = mail_cache_lookup_iter_next_record(ctx)) <= 0)

return ret;

}

/* return the next field */

file_field = *((const uint32_t *)CONST_PTR_OFFSET(ctx->rec, ctx->pos));

ctx->pos += sizeof(uint32_t);

if (file_field >= cache->file_fields_count) {

/* new field, have to re-read fields header to figure

if (!cache->locked) {

if (mail_cache_header_fields_read(cache) < 0)

return -1;

}

if (file_field >= cache->file_fields_count) {

mail_cache_set_corrupted(cache,

"field index too large (%u >= %u)",

file_field, cache->file_fields_count);

return -1;

}

/* field reading might have re-mmaped the file and

if (mail_cache_get_record(cache, ctx->offset, &ctx->rec) < 0)

return -1;

ctx->remap_counter = cache->remap_counter;

}

field_idx = cache->file_field_map[file_field];

data_size = cache->fields[field_idx].field.field_size;

if (data_size == (unsigned int)-1 &&

ctx->pos + sizeof(uint32_t) <= ctx->rec->size) {

/* variable size field. get its size from the file. */

data_size = *((const uint32_t *)

CONST_PTR_OFFSET(ctx->rec, ctx->pos));

ctx->pos += sizeof(uint32_t);

}

if (ctx->rec->size - ctx->pos < data_size) {

mail_cache_set_corrupted(cache,

"record continues outside its allocated size");

return -1;

}

field_r->field_idx = field_idx;

field_r->data = CONST_PTR_OFFSET(ctx->rec, ctx->pos);

field_r->size = data_size;

/* each record begins from 32bit aligned position */

ctx->pos += (data_size + sizeof(uint32_t)-1) & ~(sizeof(uint32_t)-1);

return 1;

}

static int mail_cache_seq(struct mail_cache_view *view, uint32_t seq)

{

struct mail_cache_lookup_iterate_ctx iter;

struct mail_cache_iterate_field field;

int ret;

if (++view->cached_exists_value == 0) {

/* wrapped, we'll have to clear the buffer */

buffer_reset(view->cached_exists_buf);

view->cached_exists_value++;

}

view->cached_exists_seq = seq;

mail_cache_lookup_iter_init(view, seq, &iter);

while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) {

buffer_write(view->cached_exists_buf, field.field_idx,

&view->cached_exists_value, 1);

}

return ret;

}

static bool

mail_cache_file_has_field(struct mail_cache *cache, unsigned int field)

{

i_assert(field < cache->fields_count);

return cache->field_file_map[field] != (uint32_t)-1;

}

int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,

unsigned int field)

{

const uint8_t *data;

i_assert(seq > 0);

if (!view->cache->opened)

(void)mail_cache_open_and_verify(view->cache);

if (!mail_cache_file_has_field(view->cache, field))

return 0;

/* FIXME: we should discard the cache if view has been synced */

if (view->cached_exists_seq != seq) {

if (mail_cache_seq(view, seq) < 0)

return -1;

}

data = view->cached_exists_buf->data;

return (field < view->cached_exists_buf->used &&

data[field] == view->cached_exists_value) ? 1 : 0;

}

bool mail_cache_field_exists_any(struct mail_cache_view *view, uint32_t seq)

{

uint32_t reset_id;

return mail_cache_lookup_cur_offset(view->view, seq, &reset_id) != 0;

}

enum mail_cache_decision_type

mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field_idx)

{

i_assert(field_idx < cache->fields_count);

return cache->fields[field_idx].field.decision;

}

static int

mail_cache_lookup_bitmask(struct mail_cache_lookup_iterate_ctx *iter,

unsigned int field_idx, unsigned int field_size,

buffer_t *dest_buf)

{

struct mail_cache_iterate_field field;

const unsigned char *src;

unsigned char *dest;

unsigned int i;

bool found = FALSE;

int ret;

/* make sure all bits are cleared first */

buffer_write_zero(dest_buf, 0, field_size);

while ((ret = mail_cache_lookup_iter_next(iter, &field)) > 0) {

if (field.field_idx != field_idx)

continue;

/* merge all bits */

src = field.data;

dest = buffer_get_space_unsafe(dest_buf, 0, field.size);

for (i = 0; i < field.size; i++)

dest[i] |= src[i];

found = TRUE;

}

return ret < 0 ? -1 : (found ? 1 : 0);

}

int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,

uint32_t seq, unsigned int field_idx)

{

const struct mail_cache_field *field_def;

struct mail_cache_lookup_iterate_ctx iter;

struct mail_cache_iterate_field field;

int ret;

ret = mail_cache_field_exists(view, seq, field_idx);

mail_cache_decision_state_update(view, seq, field_idx);

if (ret <= 0)

return ret;

/* the field should exist */

mail_cache_lookup_iter_init(view, seq, &iter);

field_def = &view->cache->fields[field_idx].field;

if (field_def->type == MAIL_CACHE_FIELD_BITMASK) {

return mail_cache_lookup_bitmask(&iter, field_idx,

field_def->field_size,

dest_buf);

}

/* return the first one that's found. if there are multiple

while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) {

if (field.field_idx == field_idx) {

buffer_append(dest_buf, field.data, field.size);

break;

}

}

return ret;

}

struct header_lookup_data {

uint32_t offset;

uint32_t data_size;

};

struct header_lookup_line {

uint32_t line_num;

struct header_lookup_data *data;

};

struct header_lookup_context {

struct mail_cache_view *view;

ARRAY_DEFINE(lines, struct header_lookup_line);

};

enum {

HDR_FIELD_STATE_DONTWANT = 0,

HDR_FIELD_STATE_WANT,

HDR_FIELD_STATE_SEEN

};

static void header_lines_save(struct header_lookup_context *ctx,

const struct mail_cache_iterate_field *field)

{

const uint32_t *lines = field->data;

uint32_t data_size = field->size;

struct header_lookup_line hdr_line;

struct header_lookup_data *hdr_data;

unsigned int i, lines_count;

/* data = { line_nums[], 0, "headers" } */

for (i = 0; data_size >= sizeof(uint32_t); i++) {

data_size -= sizeof(uint32_t);

if (lines[i] == 0)

break;

}

lines_count = i;

hdr_data = t_new(struct header_lookup_data, 1);

hdr_data->offset = (const char *)&lines[lines_count+1] -

(const char *)ctx->view->cache->data;

hdr_data->data_size = data_size;

for (i = 0; i < lines_count; i++) {

hdr_line.line_num = lines[i];

hdr_line.data = hdr_data;

array_append(&ctx->lines, &hdr_line, 1);

}

}

static int header_lookup_line_cmp(const struct header_lookup_line *l1,

const struct header_lookup_line *l2)

{

return (int)l1->line_num - (int)l2->line_num;

}

static int

mail_cache_lookup_headers_real(struct mail_cache_view *view, string_t *dest,

uint32_t seq, unsigned int field_idxs[],

unsigned int fields_count)

{

struct mail_cache *cache = view->cache;

struct mail_cache_lookup_iterate_ctx iter;

struct mail_cache_iterate_field field;

struct header_lookup_context ctx;

struct header_lookup_line *lines;

const unsigned char *p, *start, *end;

uint8_t *field_state;

unsigned int i, count, max_field = 0;

size_t hdr_size;

uint8_t want = HDR_FIELD_STATE_WANT;

buffer_t *buf;

int ret;

if (fields_count == 0)

return 1;

if (!view->cache->opened)

(void)mail_cache_open_and_verify(view->cache);

/* update the decision state regardless of whether the fields

for (i = 0; i < fields_count; i++)

mail_cache_decision_state_update(view, seq, field_idxs[i]);

/* mark all the fields we want to find. */

buf = buffer_create_dynamic(pool_datastack_create(), 32);

for (i = 0; i < fields_count; i++) {

if (!mail_cache_file_has_field(cache, field_idxs[i]))

return 0;

if (field_idxs[i] > max_field)

max_field = field_idxs[i];

buffer_write(buf, field_idxs[i], &want, 1);

}

field_state = buffer_get_modifiable_data(buf, NULL);

/* lookup the fields */

memset(&ctx, 0, sizeof(ctx));

ctx.view = view;

t_array_init(&ctx.lines, 32);

mail_cache_lookup_iter_init(view, seq, &iter);

while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) {

if (field.field_idx > max_field ||

field_state[field.field_idx] != HDR_FIELD_STATE_WANT) {

/* a) don't want it, b) duplicate */

} else {

field_state[field.field_idx] = HDR_FIELD_STATE_SEEN;

header_lines_save(&ctx, &field);

}

}

if (ret < 0)

return -1;

/* check that all fields were found */

for (i = 0; i <= max_field; i++) {

if (field_state[i] == HDR_FIELD_STATE_WANT)

return 0;

}

/* we need to return headers in the order they existed originally.

array_sort(&ctx.lines, header_lookup_line_cmp);

lines = array_get_modifiable(&ctx.lines, &count);

/* then start filling dest buffer from the headers */

for (i = 0; i < count; i++) {

start = CONST_PTR_OFFSET(cache->data, lines[i].data->offset);

end = start + lines[i].data->data_size;

/* find the end of the (multiline) header */

for (p = start; p != end; p++) {

if (*p == '\n' &&

(p+1 == end || (p[1] != ' ' && p[1] != '\t'))) {

p++;

break;

}

}

hdr_size = (size_t)(p - start);

buffer_append(dest, start, hdr_size);

/* if there are more lines for this header, the following lines

lines[i].data->offset += hdr_size;

lines[i].data->data_size -= hdr_size;

}

return 1;

}

int mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest,

uint32_t seq, unsigned int field_idxs[],

unsigned int fields_count)

{

int ret;

T_BEGIN {

ret = mail_cache_lookup_headers_real(view, dest, seq,

field_idxs, fields_count);

} T_END;

return ret;

}