src/lib-index/mail-cache-lookup.c

	mail-cache-lookup.c revision 6d8082441e480cf481bfb3f935d17fc419d789c6
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (c) 2003-2013 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainen#include "str.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-cache-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include <stdlib.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1d738cce754bc64bbc66d3355ebdaf3f6eac55f1Timo Sirainen#define CACHE_PREFETCH IO_BLOCK_SIZE
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainenint mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen              const struct mail_cache_record **rec_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    const struct mail_cache_record *rec;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    const void *data;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen    int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    i_assert(offset != 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8a4851655777e484d70b77bed6a7a8fac5c0c98cTimo Sirainen    if (offset % sizeof(uint32_t) != 0) {
8a4851655777e484d70b77bed6a7a8fac5c0c98cTimo Sirainen        /* records are always 32-bit aligned */
8a4851655777e484d70b77bed6a7a8fac5c0c98cTimo Sirainen        mail_cache_set_corrupted(cache, "invalid record offset");
8a4851655777e484d70b77bed6a7a8fac5c0c98cTimo Sirainen        return -1;
8a4851655777e484d70b77bed6a7a8fac5c0c98cTimo Sirainen    }
8a4851655777e484d70b77bed6a7a8fac5c0c98cTimo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    /* we don't know yet how large the record is, so just guess */
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH,
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen               &data) < 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    if (offset + sizeof(*rec) > cache->mmap_length) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "record points outside file");
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen    rec = data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    if (rec->size < sizeof(*rec)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "invalid record size");
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    if (rec->size > CACHE_PREFETCH) {
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        /* larger than we guessed. map the rest of the record. */
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen        if ((ret = mail_cache_map(cache, offset, rec->size, &data)) < 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen            return -1;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen        if (ret == 0) {
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen            mail_cache_set_corrupted(cache, "record points outside file");
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen            return -1;
9bd08aa09ea0cbd7b221aae9fc0534eb762d3de6Timo Sirainen        }
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen        rec = data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    *rec_r = rec;
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainenuint32_t mail_cache_lookup_cur_offset(struct mail_index_view *view,
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen                      uint32_t seq, uint32_t *reset_id_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    struct mail_cache *cache = mail_index_view_get_index(view)->cache;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    struct mail_index_map *map;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    const void *data;
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    uint32_t offset;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
03860f6dd70abfa2551a846e77a5c41cb40dc141Timo Sirainen    mail_index_lookup_ext_full(view, seq, cache->ext_id, &map, &data, NULL);
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    if (data == NULL) {
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen        /* no cache offsets */
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        return 0;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    }
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    offset = *((const uint32_t *)data);
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    if (offset == 0)
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen        return 0;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    if (!mail_index_ext_get_reset_id(view, map, cache->ext_id, reset_id_r))
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        i_unreached();
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    return offset;
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen}
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainenstatic int
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainenmail_cache_lookup_offset(struct mail_cache *cache, struct mail_index_view *view,
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen             uint32_t seq, uint32_t *offset_r)
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen{
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    uint32_t offset, reset_id;
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    int i, ret;
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    offset = mail_cache_lookup_cur_offset(view, seq, &reset_id);
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    if (offset == 0)
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen        return 0;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    /* reset_id must match file_seq or the offset is for a different cache
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen       file. if this happens, try if reopening the cache helps. if not,
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen       it was probably for an old cache file that's already lost by now. */
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    i = 0;
aff3354de83df9d683587e27461697193ff36591Timo Sirainen    while (cache->hdr->file_seq != reset_id) {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen        if (++i == 2 || reset_id < cache->hdr->file_seq)
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen            return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72368bc3058d36912317ffe55e6017205f4fa036Timo Sirainen        if (cache->locked) {
72368bc3058d36912317ffe55e6017205f4fa036Timo Sirainen            /* we're probably compressing */
72368bc3058d36912317ffe55e6017205f4fa036Timo Sirainen            return 0;
72368bc3058d36912317ffe55e6017205f4fa036Timo Sirainen        }
72368bc3058d36912317ffe55e6017205f4fa036Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        if ((ret = mail_cache_reopen(cache)) <= 0) {
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen            /* error / we already have the latest file open */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            return ret;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
6a7f64562ddd0dd2fec755ec4e9c9afde8e85cf1Timo Sirainen    *offset_r = offset;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainenbool mail_cache_track_loops(struct mail_cache_loop_track *loop_track,
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen                uoff_t offset, uoff_t size)
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen{
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    i_assert(offset != 0);
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    i_assert(size != 0);
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    /* looping happens only in rare error conditions, so it's enough if we
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen       just catch it eventually. we do this by checking if we've seen
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen       more record data than possible in the accessed file area. */
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    if (loop_track->size_sum == 0) {
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        /* first call */
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        loop_track->min_offset = offset;
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        loop_track->max_offset = offset + size;
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    } else {
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        if (loop_track->min_offset > offset)
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen            loop_track->min_offset = offset;
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        if (loop_track->max_offset < offset + size)
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen            loop_track->max_offset = offset + size;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    }
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    loop_track->size_sum += size;
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    return loop_track->size_sum >
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        (loop_track->max_offset - loop_track->min_offset);
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen}
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenvoid mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                 struct mail_cache_lookup_iterate_ctx *ctx_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_lookup_iterate_ctx *ctx = ctx_r;
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen    int ret;
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (!view->cache->opened)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        (void)mail_cache_open_and_verify(view->cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    memset(ctx, 0, sizeof(*ctx));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->view = view;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->seq = seq;
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (!MAIL_CACHE_IS_UNUSABLE(view->cache)) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        /* look up the first offset */
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen        ret = mail_cache_lookup_offset(view->cache, view->view, seq,
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen                           &ctx->offset);
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen        if (ret <= 0) {
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen            ctx->stop = TRUE;
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen            ctx->failed = ret < 0;
73a44af966c790560a72a0447f37bb35b1031a83Timo Sirainen        }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->remap_counter = view->cache->remap_counter;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    memset(&view->loop_track, 0, sizeof(view->loop_track));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen}
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainenstatic bool
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainenmail_cache_lookup_iter_transaction(struct mail_cache_lookup_iterate_ctx *ctx)
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen{
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    ctx->rec = mail_cache_transaction_lookup_rec(ctx->view->transaction,
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen                             ctx->seq,
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen                             &ctx->trans_next_idx);
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    if (ctx->rec == NULL)
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        return FALSE;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    ctx->remap_counter = ctx->view->cache->remap_counter;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    ctx->pos = sizeof(*ctx->rec);
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    ctx->rec_size = ctx->rec->size;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    return TRUE;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen}
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenstatic int
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenmail_cache_lookup_iter_next_record(struct mail_cache_lookup_iterate_ctx *ctx)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen{
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_view *view = ctx->view;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    if (ctx->failed)
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        return -1;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (ctx->rec != NULL)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        ctx->offset = ctx->rec->prev_offset;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (ctx->offset == 0) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        /* end of this record list. check newly appended data. */
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        if (view->trans_seq1 > ctx->seq ||
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            view->trans_seq2 < ctx->seq ||
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen            MAIL_CACHE_IS_UNUSABLE(view->cache))
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen            return 0;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        /* check data still in memory */
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        if (!ctx->memory_appends_checked) {
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen            if (mail_cache_lookup_iter_transaction(ctx))
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen                return 1;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen            ctx->memory_appends_checked = TRUE;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        }
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        /* check data already written to cache file */
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        if (ctx->disk_appends_checked ||
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            mail_cache_lookup_offset(view->cache, view->trans_view,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                         ctx->seq, &ctx->offset) <= 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            return 0;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        ctx->disk_appends_checked = TRUE;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        ctx->remap_counter = view->cache->remap_counter;
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        memset(&view->loop_track, 0, sizeof(view->loop_track));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen    if (ctx->stop)
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen        return 0;
d33fc6c584718efd46159e1d8f46488b9dfc66f5Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    /* look up the next record */
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    if (mail_cache_get_record(view->cache, ctx->offset, &ctx->rec) < 0)
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen        return -1;
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen    if (mail_cache_track_loops(&view->loop_track, ctx->offset,
d6693dac50e4fb547d8dc61b85820f1761a33575Timo Sirainen                   ctx->rec->size)) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        mail_cache_set_corrupted(view->cache,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                     "record list is circular");
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        return -1;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->remap_counter = view->cache->remap_counter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->pos = sizeof(*ctx->rec);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->rec_size = ctx->rec->size;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    return 1;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen}
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenint mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                struct mail_cache_iterate_field *field_r)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen{
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache *cache = ctx->view->cache;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    unsigned int field_idx;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    unsigned int data_size;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    uint32_t file_field;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    int ret;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    i_assert(ctx->remap_counter == cache->remap_counter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (ctx->pos + sizeof(uint32_t) > ctx->rec_size) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (ctx->pos != ctx->rec_size) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            mail_cache_set_corrupted(cache,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                "record has invalid size");
58febed28f2af78b2d8a281c851d9b67160c4bd3Timo Sirainen            return -1;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen        }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if ((ret = mail_cache_lookup_iter_next_record(ctx)) <= 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    /* return the next field */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    file_field = *((const uint32_t *)CONST_PTR_OFFSET(ctx->rec, ctx->pos));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->pos += sizeof(uint32_t);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (file_field >= cache->file_fields_count) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        /* new field, have to re-read fields header to figure
5e4f94015f5bbc9eefb1f2cb7af81ed50dcd2b39Timo Sirainen           out its size. don't do this if we're compressing. */
5e4f94015f5bbc9eefb1f2cb7af81ed50dcd2b39Timo Sirainen        if (!cache->locked) {
5e4f94015f5bbc9eefb1f2cb7af81ed50dcd2b39Timo Sirainen            if (mail_cache_header_fields_read(cache) < 0)
5e4f94015f5bbc9eefb1f2cb7af81ed50dcd2b39Timo Sirainen                return -1;
5e4f94015f5bbc9eefb1f2cb7af81ed50dcd2b39Timo Sirainen        }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (file_field >= cache->file_fields_count) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            mail_cache_set_corrupted(cache,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                "field index too large (%u >= %u)",
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                file_field, cache->file_fields_count);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            return -1;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        /* field reading might have re-mmaped the file and
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen           caused rec pointer to break. need to get it again. */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (mail_cache_get_record(cache, ctx->offset, &ctx->rec) < 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            return -1;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        ctx->remap_counter = cache->remap_counter;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    }
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    field_idx = cache->file_field_map[file_field];
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    data_size = cache->fields[field_idx].field.field_size;
8ae72ad7d0c69e972cfa65d1e2ce4e3e9a8b765cTimo Sirainen    if (data_size == UINT_MAX &&
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        ctx->pos + sizeof(uint32_t) <= ctx->rec->size) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        /* variable size field. get its size from the file. */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        data_size = *((const uint32_t *)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                  CONST_PTR_OFFSET(ctx->rec, ctx->pos));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        ctx->pos += sizeof(uint32_t);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (ctx->rec->size - ctx->pos < data_size) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        mail_cache_set_corrupted(cache,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            "record continues outside its allocated size");
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        return -1;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    field_r->field_idx = field_idx;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    field_r->data = CONST_PTR_OFFSET(ctx->rec, ctx->pos);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    field_r->size = data_size;
8ac66221e8fdc2c5523cff1893e0d1c5de25fa49Timo Sirainen    field_r->offset = ctx->offset + ctx->pos;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    /* each record begins from 32bit aligned position */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx->pos += (data_size + sizeof(uint32_t)-1) & ~(sizeof(uint32_t)-1);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    return 1;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstatic int mail_cache_seq(struct mail_cache_view *view, uint32_t seq)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_lookup_iterate_ctx iter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_iterate_field field;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (++view->cached_exists_value == 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        /* wrapped, we'll have to clear the buffer */
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        buffer_reset(view->cached_exists_buf);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        view->cached_exists_value++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    view->cached_exists_seq = seq;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    mail_cache_lookup_iter_init(view, seq, &iter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        buffer_write(view->cached_exists_buf, field.field_idx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                 &view->cached_exists_value, 1);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    return ret;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainenstatic bool
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainenmail_cache_file_has_field(struct mail_cache *cache, unsigned int field)
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen{
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    i_assert(field < cache->fields_count);
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    return cache->field_file_map[field] != (uint32_t)-1;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen}
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenint mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                unsigned int field)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    const uint8_t *data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    i_assert(seq > 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (!view->cache->opened)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        (void)mail_cache_open_and_verify(view->cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    if (!mail_cache_file_has_field(view->cache, field))
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    /* FIXME: we should discard the cache if view has been synced */
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    if (view->cached_exists_seq != seq) {
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        if (mail_cache_seq(view, seq) < 0)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen            return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    data = view->cached_exists_buf->data;
f65602dff8a85170176ddaa790db6df56006d132Timo Sirainen    return (field < view->cached_exists_buf->used &&
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        data[field] == view->cached_exists_value) ? 1 : 0;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainenbool mail_cache_field_exists_any(struct mail_cache_view *view, uint32_t seq)
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainen{
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainen    uint32_t reset_id;
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainen
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainen    return mail_cache_lookup_cur_offset(view->view, seq, &reset_id) != 0;
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainen}
58816241cbaf79e3f8dd7d831b7c6f02c6c38ee6Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenenum mail_cache_decision_type
710deabc6b3c305c3a842e7a2e0d173c526d13a7Timo Sirainenmail_cache_field_get_decision(struct mail_cache *cache, unsigned int field_idx)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
710deabc6b3c305c3a842e7a2e0d173c526d13a7Timo Sirainen    i_assert(field_idx < cache->fields_count);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
710deabc6b3c305c3a842e7a2e0d173c526d13a7Timo Sirainen    return cache->fields[field_idx].field.decision;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstatic int
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenmail_cache_lookup_bitmask(struct mail_cache_lookup_iterate_ctx *iter,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen              unsigned int field_idx, unsigned int field_size,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen              buffer_t *dest_buf)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_iterate_field field;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    const unsigned char *src;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    unsigned char *dest;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    unsigned int i;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    bool found = FALSE;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    int ret;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    /* make sure all bits are cleared first */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    buffer_write_zero(dest_buf, 0, field_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    while ((ret = mail_cache_lookup_iter_next(iter, &field)) > 0) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (field.field_idx != field_idx)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            continue;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        /* merge all bits */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        src = field.data;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        dest = buffer_get_space_unsafe(dest_buf, 0, field.size);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        for (i = 0; i < field.size; i++)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            dest[i] |= src[i];
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        found = TRUE;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    return ret < 0 ? -1 : (found ? 1 : 0);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen}
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainenint mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
710deabc6b3c305c3a842e7a2e0d173c526d13a7Timo Sirainen                uint32_t seq, unsigned int field_idx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    const struct mail_cache_field *field_def;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_lookup_iterate_ctx iter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_iterate_field field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    int ret;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen    ret = mail_cache_field_exists(view, seq, field_idx);
710deabc6b3c305c3a842e7a2e0d173c526d13a7Timo Sirainen    mail_cache_decision_state_update(view, seq, field_idx);
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen    if (ret <= 0)
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen        return ret;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen    /* the field should exist */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    mail_cache_lookup_iter_init(view, seq, &iter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    field_def = &view->cache->fields[field_idx].field;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if (field_def->type == MAIL_CACHE_FIELD_BITMASK) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        return mail_cache_lookup_bitmask(&iter, field_idx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                         field_def->field_size,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                         dest_buf);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    /* return the first one that's found. if there are multiple
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen       they're all identical. */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (field.field_idx == field_idx) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            buffer_append(dest_buf, field.data, field.size);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            break;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainenstruct header_lookup_data {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t data_size;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    const unsigned char *data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainenstruct header_lookup_line {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t line_num;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    struct header_lookup_data *data;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen};
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenstruct header_lookup_context {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_view *view;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    pool_t pool;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen    ARRAY(struct header_lookup_line) lines;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen};
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainenenum {
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    HDR_FIELD_STATE_DONTWANT = 0,
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    HDR_FIELD_STATE_WANT,
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    HDR_FIELD_STATE_SEEN
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenstatic void header_lines_save(struct header_lookup_context *ctx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen                  const struct mail_cache_iterate_field *field)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    const uint32_t *lines = field->data;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    uint32_t data_size = field->size;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    struct header_lookup_line hdr_line;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        struct header_lookup_data *hdr_data;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    void *data_dup;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    unsigned int i, lines_count, pos;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* data = { line_nums[], 0, "headers" } */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; data_size >= sizeof(uint32_t); i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        data_size -= sizeof(uint32_t);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (lines[i] == 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            break;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    lines_count = i;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    pos = (lines_count+1) * sizeof(uint32_t);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    hdr_data = p_new(ctx->pool, struct header_lookup_data, 1);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    hdr_data->data_size = data_size;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    hdr_data->data = data_dup = data_size == 0 ? NULL :
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen        p_malloc(ctx->pool, data_size);
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    memcpy(data_dup, CONST_PTR_OFFSET(field->data, pos), data_size);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; i < lines_count; i++) {
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        hdr_line.line_num = lines[i];
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        hdr_line.data = hdr_data;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        array_append(&ctx->lines, &hdr_line, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainenstatic int header_lookup_line_cmp(const struct header_lookup_line *l1,
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen                  const struct header_lookup_line *l2)
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen{
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    return (int)l1->line_num - (int)l2->line_num;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainenstatic int
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainenmail_cache_lookup_headers_real(struct mail_cache_view *view, string_t *dest,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen                   uint32_t seq, unsigned int field_idxs[],
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen                   unsigned int fields_count, pool_t *pool_r)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct mail_cache *cache = view->cache;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_lookup_iterate_ctx iter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_iterate_field field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct header_lookup_context ctx;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    struct header_lookup_line *lines;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    const unsigned char *p, *start, *end;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    uint8_t *field_state;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    unsigned int i, count, max_field = 0;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    size_t hdr_size;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    uint8_t want = HDR_FIELD_STATE_WANT;
65a5fb2343f9870713bfca0b24abb58dcade605eTimo Sirainen    buffer_t *buf;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    int ret;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    *pool_r = NULL;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (fields_count == 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return 1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
2a21435425c50514fa4d0708110a4ed7f0d85661Timo Sirainen    if (!view->cache->opened)
2a21435425c50514fa4d0708110a4ed7f0d85661Timo Sirainen        (void)mail_cache_open_and_verify(view->cache);
2a21435425c50514fa4d0708110a4ed7f0d85661Timo Sirainen
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen    /* update the decision state regardless of whether the fields
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen       actually exist or not. */
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen    for (i = 0; i < fields_count; i++)
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen        mail_cache_decision_state_update(view, seq, field_idxs[i]);
0beb5d6c661ee68130a954ed0f31a34c19195fb7Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    /* mark all the fields we want to find. */
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 32);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; i < fields_count; i++) {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen        if (!mail_cache_file_has_field(cache, field_idxs[i]))
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (field_idxs[i] > max_field)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            max_field = field_idxs[i];
65a5fb2343f9870713bfca0b24abb58dcade605eTimo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        buffer_write(buf, field_idxs[i], &want, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    field_state = buffer_get_modifiable_data(buf, NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    /* lookup the fields */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    memset(&ctx, 0, sizeof(ctx));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx.view = view;
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    ctx.pool = *pool_r = pool_alloconly_create("mail cache headers", 1024);
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    t_array_init(&ctx.lines, 32);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    mail_cache_lookup_iter_init(view, seq, &iter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        if (field.field_idx > max_field ||
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            field_state[field.field_idx] != HDR_FIELD_STATE_WANT) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            /* a) don't want it, b) duplicate */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        } else {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            field_state[field.field_idx] = HDR_FIELD_STATE_SEEN;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            header_lines_save(&ctx, &field);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    }
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen    if (ret < 0)
84004c37192dc91a8dcc9f213ca2cfa22b4f40e4Timo Sirainen        return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* check that all fields were found */
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    for (i = 0; i <= max_field; i++) {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen        if (field_state[i] == HDR_FIELD_STATE_WANT)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    /* we need to return headers in the order they existed originally.
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen       we can do this by sorting the messages by their line numbers. */
c9dea5c23355dea35c6fa423de69f6507852efe4Timo Sirainen    array_sort(&ctx.lines, header_lookup_line_cmp);
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    lines = array_get_modifiable(&ctx.lines, &count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* then start filling dest buffer from the headers */
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen    for (i = 0; i < count; i++) {
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen        start = lines[i].data->data;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        end = start + lines[i].data->data_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        /* find the end of the (multiline) header */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        for (p = start; p != end; p++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            if (*p == '\n' &&
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                (p+1 == end || (p[1] != ' ' && p[1] != '\t'))) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                p++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                break;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        hdr_size = (size_t)(p - start);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        buffer_append(dest, start, hdr_size);
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        /* if there are more lines for this header, the following lines
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen           continue after this one. so skip this line. */
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen        lines[i].data->data += hdr_size;
1b97a59edb073e9a89ac43a21a9abe5d590d4a56Timo Sirainen        lines[i].data->data_size -= hdr_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    return 1;
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen}
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainenint mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest,
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen                  uint32_t seq, unsigned int field_idxs[],
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen                  unsigned int fields_count)
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen{
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen    pool_t pool;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen    int ret;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen    T_BEGIN {
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen        ret = mail_cache_lookup_headers_real(view, dest, seq,
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen                             field_idxs, fields_count,
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen                             &pool);
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen        if (pool != NULL)
6d8082441e480cf481bfb3f935d17fc419d789c6Timo Sirainen            pool_unref(&pool);
19e8adccba16ff419f5675b1575358c2956dce83Timo Sirainen    } T_END;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen    return ret;
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen}