src/lib-index/mail-cache-lookup.c

	mail-cache-lookup.c revision 6a8a4c9f530668cd8961b73d702856ed94f05f80
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainen#include "str.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-cache-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#include <stdlib.h>
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen#define CACHE_PREFETCH 1024
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainenint mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen              const struct mail_cache_record **rec_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen    const struct mail_cache_record *cache_rec;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen    *rec_r = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (offset == 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    if (mail_cache_map(cache, offset,
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen               sizeof(*cache_rec) + CACHE_PREFETCH) < 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (offset + sizeof(*cache_rec) > cache->mmap_length) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "record points outside file");
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    cache_rec = CACHE_RECORD(cache, offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    if (cache_rec->size < sizeof(*cache_rec)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "invalid record size");
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    if (cache_rec->size > CACHE_PREFETCH) {
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen        if (mail_cache_map(cache, offset, cache_rec->size) < 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen            return -1;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        cache_rec = CACHE_RECORD(cache, offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    if (cache_rec->size > cache->mmap_length ||
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen        offset + cache_rec->size > cache->mmap_length) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_corrupted(cache, "record points outside file");
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen    *rec_r = cache_rec;
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenstatic int
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenmail_cache_lookup_offset(struct mail_cache *cache, struct mail_index_view *view,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen             uint32_t seq, uint32_t *offset_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    struct mail_index_map *map;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    const struct mail_index_ext *ext;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    const void *data;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    uint32_t idx;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    int i, ret;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    if (mail_index_lookup_ext_full(view, seq, cache->ext_id,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen                       &map, &data) < 0)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        return -1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    if (data == NULL)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        return 0;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    if (!mail_index_map_get_ext_idx(map, cache->ext_id, &idx)) {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        /* no cache */
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        return 0;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    }
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen    ext = array_idx(&map->extensions, idx);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    for (i = 0; i < 2; i++) {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        if (cache->hdr->file_seq == ext->reset_id) {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen            *offset_r = *((const uint32_t *)data);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            return 1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        if ((ret = mail_cache_reopen(cache)) <= 0)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            return ret;
4b0a728d4e5ad007b5cee2043a3d2030359f6613Timo Sirainen
4b0a728d4e5ad007b5cee2043a3d2030359f6613Timo Sirainen        if (MAIL_CACHE_IS_UNUSABLE(cache))
4b0a728d4e5ad007b5cee2043a3d2030359f6613Timo Sirainen            return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainenstatic int
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainenmail_cache_foreach_rec(struct mail_cache_view *view, uint32_t *offset,
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen               mail_cache_foreach_callback_t *callback, void *context)
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct mail_cache *cache = view->cache;
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen    const struct mail_cache_record *cache_rec;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    size_t pos, next_pos, max_size;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    unsigned int data_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t file_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    unsigned int field;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    int ret;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen    if (mail_cache_get_record(view->cache, *offset, &cache_rec) < 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen        return -1;
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen    if (cache_rec == NULL) {
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen        *offset = 0;
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen        return 1;
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen    }
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    max_size = cache_rec->size;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    if (max_size < sizeof(*cache_rec) + sizeof(uint32_t)*2) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        mail_cache_set_corrupted(cache, "record has invalid size");
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        return -1;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    }
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    max_size -= sizeof(uint32_t);
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    for (pos = sizeof(*cache_rec); pos < max_size; ) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        file_field =
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            *((const uint32_t *)CONST_PTR_OFFSET(cache_rec, pos));
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        pos += sizeof(uint32_t);
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (file_field >= cache->file_fields_count) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            /* new field, have to re-read fields header to figure
325d4ad220bd13f6d176391d962a0e33c856a7f6Timo Sirainen               out its size */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            if (mail_cache_header_fields_read(cache) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            if (file_field >= cache->file_fields_count) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                mail_cache_set_corrupted(cache,
68aa521f6fc7a40480a56a7b3d61c7dda19061f3Timo Sirainen                    "field index too large (%u >= %u)",
68aa521f6fc7a40480a56a7b3d61c7dda19061f3Timo Sirainen                    file_field, cache->file_fields_count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            }
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen            /* field reading might have re-mmaped the file and
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen               caused cache_rec to break. need to get it again. */
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen            if (mail_cache_get_record(view->cache, *offset,
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen                          &cache_rec) < 0)
fcfb528483369975066c6adf1c55c16e6fb6e91fTimo Sirainen                return -1;
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen            i_assert(cache_rec != NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        field = cache->file_field_map[file_field];
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen        data_size = cache->fields[field].field.field_size;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        if (data_size == (unsigned int)-1) {
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen            data_size = *((const uint32_t *)
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen                      CONST_PTR_OFFSET(cache_rec, pos));
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen            pos += sizeof(uint32_t);
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        }
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        next_pos = pos + ((data_size + 3) & ~3);
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen        if (data_size > cache_rec->size || next_pos > cache_rec->size) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            mail_cache_set_corrupted(cache,
325d4ad220bd13f6d176391d962a0e33c856a7f6Timo Sirainen                "record continues outside its allocated size");
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen            return -1;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        }
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen        ret = callback(view, field, CONST_PTR_OFFSET(cache_rec, pos),
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen                   data_size, context);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (ret != 1)
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen            return ret;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        pos = next_pos;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    }
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen    *offset = cache_rec->prev_offset;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    return 1;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen}
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenstatic bool find_offset(struct mail_cache_view *view, uint32_t offset)
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen{
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    const uint32_t *offsets;
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen    unsigned int i, count;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen    offsets = array_get(&view->tmp_offsets, &count);
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen    for (i = 0; i < count; i++) {
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen        if (offsets[i] == offset)
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen            return TRUE;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    }
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    return FALSE;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen}
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenint mail_cache_foreach(struct mail_cache_view *view, uint32_t seq,
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen                       mail_cache_foreach_callback_t *callback, void *context)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen    uint32_t offset;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    int ret;
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (!view->cache->opened)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        (void)mail_cache_open_and_verify(view->cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen        if (MAIL_CACHE_IS_UNUSABLE(view->cache))
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        return 0;
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (view->cached_offset_seq == seq)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        offset = view->cached_offset;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    else {
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        if ((ret = mail_cache_lookup_offset(view->cache, view->view,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen                            seq, &offset)) <= 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            return ret;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        view->cached_offset_seq = seq;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        view->cached_offset = offset;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    ret = 1;
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen    array_clear(&view->tmp_offsets);
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    while (offset != 0 && ret > 0) {
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen        if (find_offset(view, offset)) {
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen            mail_cache_set_corrupted(view->cache,
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen                         "record list is circular");
58febed28f2af78b2d8a281c851d9b67160c4bd3Timo Sirainen            return -1;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen        }
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen        array_append(&view->tmp_offsets, &offset, 1);
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen        ret = mail_cache_foreach_rec(view, &offset,
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen                         callback, context);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    if (ret > 0 && view->trans_seq1 <= seq && view->trans_seq2 >= seq &&
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen        mail_cache_lookup_offset(view->cache, view->trans_view,
4b0a728d4e5ad007b5cee2043a3d2030359f6613Timo Sirainen                     seq, &offset) > 0) {
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen        array_clear(&view->tmp_offsets);
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen        while (offset != 0 && ret > 0) {
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen            if (find_offset(view, offset)) {
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen                mail_cache_set_corrupted(view->cache,
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen                    "record list is circular");
58febed28f2af78b2d8a281c851d9b67160c4bd3Timo Sirainen                return -1;
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen            }
287ba82a8da3eaa473b5735d4eeac2fb4c5d8117Timo Sirainen            array_append(&view->tmp_offsets, &offset, 1);
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen            ret = mail_cache_foreach_rec(view, &offset,
2112c190034c937b4b1613c3689d8239adb6c310Timo Sirainen                             callback, context);
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen        }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    }
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen
785b1ca149341b70bf2cb8cc3049f1c4c1070b52Timo Sirainen    return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic int
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainenmail_cache_seq_callback(struct mail_cache_view *view, uint32_t field,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            const void *data __attr_unused__,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            size_t data_size __attr_unused__,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            void *context __attr_unused__)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    buffer_write(view->cached_exists_buf, field,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen             &view->cached_exists_value, 1);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    return 1;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstatic int mail_cache_seq(struct mail_cache_view *view, uint32_t seq)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (++view->cached_exists_value == 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        /* wrapped, we'll have to clear the buffer */
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen        memset(buffer_get_modifiable_data(view->cached_exists_buf,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                          NULL), 0,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen               buffer_get_size(view->cached_exists_buf));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        view->cached_exists_value++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    view->cached_exists_seq = seq;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    ret = mail_cache_foreach(view, seq, mail_cache_seq_callback, NULL);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    return ret < 0 ? -1 : 0;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenint mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                unsigned int field)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    const uint8_t *data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t file_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    size_t size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    i_assert(seq > 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    i_assert(field < view->cache->fields_count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen    if (!view->cache->opened)
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen        (void)mail_cache_open_and_verify(view->cache);
6a8a4c9f530668cd8961b73d702856ed94f05f80Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    file_field = view->cache->field_file_map[field];
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (file_field == (uint32_t)-1)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    if (view->cached_exists_seq != seq) {
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        if (mail_cache_seq(view, seq) < 0)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen            return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    data = buffer_get_data(view->cached_exists_buf, &size);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    return size <= field ? FALSE :
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen        data[field] == view->cached_exists_value;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenenum mail_cache_decision_type
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenmail_cache_field_get_decision(struct mail_cache *cache, unsigned int field)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    i_assert(field < cache->fields_count);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen    return cache->fields[field].field.decision;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen}
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstruct mail_cache_lookup_context {
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    buffer_t *dest_buf;
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    uint32_t field;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen    bool found;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstatic int
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenmail_cache_lookup_callback(struct mail_cache_view *view __attr_unused__,
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen               uint32_t field, const void *data,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen               size_t data_size, void *context)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen{
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        struct mail_cache_lookup_context *ctx = context;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    if (ctx->field != field)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        return 1;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    buffer_append(ctx->dest_buf, data, data_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    ctx->found = TRUE;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainenstatic int
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainenmail_cache_lookup_bitmask_callback(struct mail_cache_view *view __attr_unused__,
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen                   uint32_t field, const void *data,
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen                   size_t data_size, void *context)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen{
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        struct mail_cache_lookup_context *ctx = context;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    unsigned char *dest;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    size_t i;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen        if (ctx->field != field)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        return 1;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    /* merge all bits */
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    dest = buffer_get_space_unsafe(ctx->dest_buf, 0, data_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    for (i = 0; i < data_size; i++)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        dest[i] |= ((const unsigned char *)data)[i];
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    ctx->found = TRUE;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    return 1;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen}
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainenint mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                uint32_t seq, unsigned int field)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct mail_cache_lookup_context ctx;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    unsigned int data_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    int ret;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if ((ret = mail_cache_field_exists(view, seq, field)) <= 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return ret;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    mail_cache_decision_lookup(view, seq, field);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    /* should exist. find it. */
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    ctx.field = field;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    ctx.dest_buf = dest_buf;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    ctx.found = FALSE;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    if (view->cache->fields[field].field.type != MAIL_CACHE_FIELD_BITMASK) {
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        ret = mail_cache_foreach(view, seq, mail_cache_lookup_callback,
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen                     &ctx);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    } else {
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        /* make sure we're cleared first */
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        data_size = view->cache->fields[field].field.field_size;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        memset(buffer_get_space_unsafe(dest_buf, 0, data_size),
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen               0, data_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        ret = mail_cache_foreach(view, seq,
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen                     mail_cache_lookup_bitmask_callback,
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen                     &ctx);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    }
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    return ret < 0 ? -1 : ctx.found;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstruct header_lookup_data_rec {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t offset;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t data_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstruct header_lookup_data {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint32_t line_num;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        struct header_lookup_data_rec *data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstruct header_lookup_context {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    unsigned int *fields;
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen    unsigned int fields_count;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    buffer_t *data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    unsigned int max_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint8_t *fields_found;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic int
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainenheaders_find_callback(struct mail_cache_view *view, uint32_t field,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen              const void *data, size_t data_size, void *context)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct header_lookup_context *ctx = context;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    const uint32_t *lines = data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct header_lookup_data hdr_data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        struct header_lookup_data_rec *hdr_data_rec;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    unsigned int i, lines_count;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    if (field > ctx->max_field || ctx->fields_found[field] != 1) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        /* a) don't want it, b) duplicate */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return 1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen    ctx->fields_found[field]++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* data = { line_nums[], 0, "headers" } */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; data_size >= sizeof(uint32_t); i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        data_size -= sizeof(uint32_t);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (lines[i] == 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            break;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    lines_count = i;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    hdr_data_rec = t_new(struct header_lookup_data_rec, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    hdr_data_rec->offset = (const char *)&lines[lines_count+1] -
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        (const char *)view->cache->data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    hdr_data_rec->data_size = (uint32_t)data_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; i < lines_count; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        hdr_data.line_num = lines[i];
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        hdr_data.data = hdr_data_rec;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        buffer_append(ctx->data, &hdr_data, sizeof(hdr_data));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic int header_lookup_data_cmp(const void *p1, const void *p2)
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    const struct header_lookup_data *d1 = p1, *d2 = p2;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    return (int)d1->line_num - (int)d2->line_num;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenint mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                  uint32_t seq, unsigned int fields[],
9d3ccd79130199ffdb19a688027d49bf20a4aaaaTimo Sirainen                  unsigned int fields_count)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct mail_cache *cache = view->cache;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct header_lookup_context ctx;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    struct header_lookup_data *data;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    const unsigned char *p, *start, *end;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    size_t i, size, hdr_size;
65a5fb2343f9870713bfca0b24abb58dcade605eTimo Sirainen    uint8_t one = 1;
65a5fb2343f9870713bfca0b24abb58dcade605eTimo Sirainen    buffer_t *buf;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    int ret;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (fields_count == 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return 1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    t_push();
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* @UNSAFE */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    memset(&ctx, 0, sizeof(ctx));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    ctx.fields = t_new(unsigned int, fields_count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    ctx.fields_count = fields_count;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
65a5fb2343f9870713bfca0b24abb58dcade605eTimo Sirainen    ctx.max_field = 0;
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    buf = buffer_create_dynamic(pool_datastack_create(), 32);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; i < fields_count; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        i_assert(fields[i] < cache->fields_count);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen        if (cache->field_file_map[fields[i]] == (unsigned int)-1) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            /* not cached at all */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            t_pop();
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen        if (fields[i] > ctx.max_field)
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen            ctx.max_field = fields[i];
65a5fb2343f9870713bfca0b24abb58dcade605eTimo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen        buffer_write(buf, fields[i], &one, 1);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen                ctx.fields[i] = fields[i];
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen    ctx.fields_found = buffer_get_modifiable_data(buf, NULL);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    ctx.data = buffer_create_dynamic(pool_datastack_create(), 256);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* we need to return them in sorted order. create array:
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen       { line number -> cache file offset } */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    ret = mail_cache_foreach(view, seq, headers_find_callback, &ctx);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (ret <= 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        t_pop();
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return ret;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* check that all fields were found */
a11af9df3bb29db35f956d337ab88160729580b0Timo Sirainen    for (i = 0; i <= ctx.max_field; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (ctx.fields_found[i] == 1) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            t_pop();
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen    data = buffer_get_modifiable_data(ctx.data, &size);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    size /= sizeof(*data);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    qsort(data, size, sizeof(*data), header_lookup_data_cmp);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* then start filling dest buffer from the headers */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    for (i = 0; i < size; i++) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen        start = CONST_PTR_OFFSET(cache->data, data[i].data->offset);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        end = start + data[i].data->data_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        for (p = start; p != end; p++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            if (*p == '\n' &&
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                (p+1 == end || (p[1] != ' ' && p[1] != '\t'))) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                p++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                break;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        hdr_size = (size_t)(p - start);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        data[i].data->offset += hdr_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        data[i].data->data_size += hdr_size;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        buffer_append(dest, start, hdr_size);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    t_pop();
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    return 1;
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen}