src/lib-index/mail-cache-fields.c

	mail-cache-fields.c revision a75d470c9223a75801418fcdda258885c36317e0
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen/* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "lib.h"
bdd36cfdba3ff66d25570a9ff568d69e1eb543cfTimo Sirainen#include "ioloop.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "buffer.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "hash.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "file-cache.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "read-full.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "write-full.h"
6d2b3ce2c6ef62334985ece4f0ab8b154e0e9560Timo Sirainen#include "mmap-util.h"
c6335901c67a4c9365319190a111a2168f3b06f5Timo Sirainen#include "mail-cache-private.h"
c6335901c67a4c9365319190a111a2168f3b06f5Timo Sirainen
c6335901c67a4c9365319190a111a2168f3b06f5Timo Sirainen#include <stddef.h>
01230de017cd273de41143d88e9c18df1243ae8aTimo Sirainen
01230de017cd273de41143d88e9c18df1243ae8aTimo Sirainen#define CACHE_FIELD_IS_NEWLY_WANTED(cache, field_idx) \
047c00cd3f7f403672f81569413669238df8c15aTimo Sirainen    ((cache)->field_file_map[field_idx] == (uint32_t)-1 && \
1f9d1bedae25d86f26c239055c5487499dfeeb58Timo Sirainen     (cache)->fields[field_idx].used)
047c00cd3f7f403672f81569413669238df8c15aTimo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic bool field_has_fixed_size(enum mail_cache_field_type type)
f37ecd72aad9b806aae83f71bacafdce32146945Timo Sirainen{
f37ecd72aad9b806aae83f71bacafdce32146945Timo Sirainen    switch (type) {
e2bdca8201e4aa1cd31332ffbdd4c6eef9151d5eTimo Sirainen    case MAIL_CACHE_FIELD_FIXED_SIZE:
e2bdca8201e4aa1cd31332ffbdd4c6eef9151d5eTimo Sirainen    case MAIL_CACHE_FIELD_BITMASK:
e2bdca8201e4aa1cd31332ffbdd4c6eef9151d5eTimo Sirainen        return TRUE;
32b78da5dfbbf6a06b3dbdc9278c60b55714f9bcTimo Sirainen    case MAIL_CACHE_FIELD_VARIABLE_SIZE:
32b78da5dfbbf6a06b3dbdc9278c60b55714f9bcTimo Sirainen    case MAIL_CACHE_FIELD_STRING:
32b78da5dfbbf6a06b3dbdc9278c60b55714f9bcTimo Sirainen    case MAIL_CACHE_FIELD_HEADER:
32b78da5dfbbf6a06b3dbdc9278c60b55714f9bcTimo Sirainen        return FALSE;
8b5c520883aa37bb55646286d375fdbae294d710Timo Sirainen
8b5c520883aa37bb55646286d375fdbae294d710Timo Sirainen    case MAIL_CACHE_FIELD_COUNT:
8b5c520883aa37bb55646286d375fdbae294d710Timo Sirainen        break;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    i_unreached();
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
f7423cbbd9dea363a5df18ebb96da055a977ae79Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic bool field_decision_is_valid(enum mail_cache_decision_type type)
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    switch (type & ~MAIL_CACHE_DECISION_FORCED) {
7487ff578435377bbeefffdbfb78ca09ed1292dfTimo Sirainen    case MAIL_CACHE_DECISION_NO:
89237470342ea6d4bbdf4cff9764037cfb3f6f45Timo Sirainen    case MAIL_CACHE_DECISION_TEMP:
89237470342ea6d4bbdf4cff9764037cfb3f6f45Timo Sirainen    case MAIL_CACHE_DECISION_YES:
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen        return TRUE;
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen    default:
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        return FALSE;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainenstatic int field_type_verify(struct mail_cache *cache, unsigned int idx,
4d4d6d4745682790c20d759ba93dbea46b812c5dTimo Sirainen                 enum mail_cache_field_type type, unsigned int size)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    const struct mail_cache_field *field = &cache->fields[idx].field;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    if (field->type != type) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen        mail_cache_set_corrupted(cache,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen            "registered field %s type changed", field->name);
a3fe8c0c54d87822f4b4f8f0d10caac611861b2bTimo Sirainen        return -1;
578ef2538ccf42e2a48234c24a8b709397101d88Timo Sirainen    }
578ef2538ccf42e2a48234c24a8b709397101d88Timo Sirainen    if (field->field_size != size && field_has_fixed_size(type)) {
578ef2538ccf42e2a48234c24a8b709397101d88Timo Sirainen        mail_cache_set_corrupted(cache,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen            "registered field %s size changed", field->name);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen        return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    return 0;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen}
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic void
d5eb47a791ec56149fd711cd8e44efc8babeaae5Timo Sirainenmail_cache_field_update(struct mail_cache *cache,
d5eb47a791ec56149fd711cd8e44efc8babeaae5Timo Sirainen            const struct mail_cache_field *newfield)
d5eb47a791ec56149fd711cd8e44efc8babeaae5Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    struct mail_cache_field_private *orig;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
e0740628f6ca05f4bc79a9d8a90b650f4d38d4d0Timo Sirainen    i_assert(newfield->type < MAIL_CACHE_FIELD_COUNT);
e0740628f6ca05f4bc79a9d8a90b650f4d38d4d0Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    orig = &cache->fields[newfield->idx];
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen    if (newfield->decision != MAIL_CACHE_DECISION_NO &&
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen        orig->field.decision != newfield->decision) {
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen        orig->field.decision = newfield->decision;
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen        orig->decision_dirty = TRUE;
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen    }
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen    if (orig->field.last_used < newfield->last_used) {
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen        orig->field.last_used = newfield->last_used;
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen        orig->decision_dirty = TRUE;
86791365b10f45982c88e70f2eb94fd6c3fea151Timo Sirainen    }
ccf50662cc02b5e703039a4ff7f91a4470e25b71Timo Sirainen    if (orig->decision_dirty)
ccf50662cc02b5e703039a4ff7f91a4470e25b71Timo Sirainen        cache->field_header_write_pending = TRUE;
ccf50662cc02b5e703039a4ff7f91a4470e25b71Timo Sirainen
ccf50662cc02b5e703039a4ff7f91a4470e25b71Timo Sirainen    (void)field_type_verify(cache, newfield->idx,
ccf50662cc02b5e703039a4ff7f91a4470e25b71Timo Sirainen                newfield->type, newfield->field_size);
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen}
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainenvoid mail_cache_register_fields(struct mail_cache *cache,
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen                struct mail_cache_field *fields,
6fdfa4d4cf14d1d7764d7faa8258f112e39c8dbeTimo Sirainen                unsigned int fields_count)
3f603ef00e35fca21605afa0ad8d76e94fee2b96Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    char *name;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    void *value;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    unsigned int new_idx;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen    unsigned int i, j;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
1ffb2afe6d7e8860a2231a4827078cf2ef9c22cdTimo Sirainen    new_idx = cache->fields_count;
1ffb2afe6d7e8860a2231a4827078cf2ef9c22cdTimo Sirainen    for (i = 0; i < fields_count; i++) {
1ffb2afe6d7e8860a2231a4827078cf2ef9c22cdTimo Sirainen        if (hash_table_lookup_full(cache->field_name_hash,
0161376aac025266d8654577c4b9ce371ffc87eaTimo Sirainen                       fields[i].name, &name, &value)) {
0161376aac025266d8654577c4b9ce371ffc87eaTimo Sirainen            fields[i].idx = POINTER_CAST_TO(value, unsigned int);
0161376aac025266d8654577c4b9ce371ffc87eaTimo Sirainen            mail_cache_field_update(cache, &fields[i]);
0161376aac025266d8654577c4b9ce371ffc87eaTimo Sirainen            continue;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        /* check if the same header is being registered in the
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen           same field array */
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        for (j = 0; j < i; j++) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen            if (strcasecmp(fields[i].name, fields[j].name) == 0) {
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen                fields[i].idx = fields[j].idx;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen                break;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen            }
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen        if (j == i)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen            fields[i].idx = new_idx++;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    if (new_idx == cache->fields_count)
b2d562f9c7fd13f9a16e9b3bcee904630b80b1feTimo Sirainen        return;
b2d562f9c7fd13f9a16e9b3bcee904630b80b1feTimo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    /* @UNSAFE */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen    cache->fields = i_realloc(cache->fields,
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen                  cache->fields_count * sizeof(*cache->fields),
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen                  new_idx * sizeof(*cache->fields));
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen    cache->field_file_map =
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen        i_realloc(cache->field_file_map,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen              cache->fields_count * sizeof(*cache->field_file_map),
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen              new_idx * sizeof(*cache->field_file_map));
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen
275385a2ecc58e41dc7df3ce3cd943caaa58c4d1Timo Sirainen    for (i = 0; i < fields_count; i++) {
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen        unsigned int idx = fields[i].idx;
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen        if (idx < cache->fields_count)
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen            continue;
6c2ce1d5bf17b21e804a079eb0f973b7ab83e0d8Timo Sirainen
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen        /* new index - save it */
db693bf6fcae96d834567f1782257517b7207655Timo Sirainen        name = p_strdup(cache->field_pool, fields[i].name);
21e6b4fd844fd074583b17f09e1f27b9835ee238Timo Sirainen        cache->fields[idx].field = fields[i];
21e6b4fd844fd074583b17f09e1f27b9835ee238Timo Sirainen        cache->fields[idx].field.name = name;
21e6b4fd844fd074583b17f09e1f27b9835ee238Timo Sirainen        cache->fields[idx].field.last_used = fields[i].last_used;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        cache->field_file_map[idx] = (uint32_t)-1;
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen        if (!field_has_fixed_size(cache->fields[idx].field.type))
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen            cache->fields[idx].field.field_size = (unsigned int)-1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen        hash_table_insert(cache->field_name_hash, name,
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen                  POINTER_CAST(idx));
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen    }
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen    cache->fields_count = new_idx;
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen}
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainenunsigned int
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenmail_cache_register_lookup(struct mail_cache *cache, const char *name)
{
    char *key;
    void *value;

    if (hash_table_lookup_full(cache->field_name_hash, name, &key, &value))
        return POINTER_CAST_TO(value, unsigned int);
    else
        return -1U;
}

const struct mail_cache_field *
mail_cache_register_get_field(struct mail_cache *cache, unsigned int field_idx)
{
    i_assert(field_idx < cache->fields_count);

    return &cache->fields[field_idx].field;
}

const struct mail_cache_field *
mail_cache_register_get_list(struct mail_cache *cache, pool_t pool,
                 unsigned int *count_r)
{
        struct mail_cache_field *list;
    unsigned int i;

    if (!cache->opened)
        (void)mail_cache_open_and_verify(cache);

    list = cache->fields_count == 0 ? NULL :
        p_new(pool, struct mail_cache_field, cache->fields_count);
    for (i = 0; i < cache->fields_count; i++) {
        list[i] = cache->fields[i].field;
        list[i].name = p_strdup(pool, list[i].name);
    }

    *count_r = cache->fields_count;
    return list;
}

static int mail_cache_header_fields_get_offset(struct mail_cache *cache,
                           uint32_t *offset_r, bool map)
{
    const struct mail_cache_header_fields *field_hdr;
    struct mail_cache_header_fields tmp_field_hdr;
    uint32_t offset = 0, next_offset;
    unsigned int next_count = 0;
    bool invalidate = FALSE;
    int ret;

    if (MAIL_CACHE_IS_UNUSABLE(cache)) {
        *offset_r = 0;
        return 0;
    }

    /* find the latest header */
    offset = 0;
    next_offset = cache->last_field_header_offset != 0 ?
        cache->last_field_header_offset :
        mail_index_offset_to_uint32(cache->hdr->field_header_offset);
    while (next_offset != 0) {
        if (next_offset == offset) {
            mail_cache_set_corrupted(cache,
                "next_offset in field header loops");
            return -1;
        }
        offset = next_offset;
        invalidate = TRUE;

        if (cache->mmap_base != NULL) {
            if (mail_cache_map(cache, offset,
                       sizeof(*field_hdr)) < 0)
                return -1;
            if (offset >= cache->mmap_length) {
                mail_cache_set_corrupted(cache,
                    "header field next_offset points outside file");
                return -1;
            }

            field_hdr = CONST_PTR_OFFSET(cache->data, offset);
        } else {
            /* if we need to follow multiple offsets to get to
               the last one, it's faster to just pread() the file
               instead of going through cache */
            ret = pread_full(cache->fd, &tmp_field_hdr,
                     sizeof(tmp_field_hdr), offset);
            if (ret < 0) {
                mail_cache_set_syscall_error(cache, "pread()");
                return -1;
            }
            if (ret == 0) {
                mail_cache_set_corrupted(cache,
                    "header field next_offset points outside file");
                return -1;
            }
            field_hdr = &tmp_field_hdr;
        }

        next_offset =
            mail_index_offset_to_uint32(field_hdr->next_offset);
        next_count++;
    }

    if (offset == 0) {
        mail_cache_set_corrupted(cache, "missing header fields");
        return -1;
    }
    cache->last_field_header_offset = offset;

    if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT)
        cache->need_compress_file_seq = cache->hdr->file_seq;

    if (map) {
        if (cache->file_cache != NULL && invalidate) {
            /* if this isn't the first header in file and we hadn't
               read this before, we can't trust that the cached
               data is valid */
            file_cache_invalidate(cache->file_cache, offset,
                          field_hdr->size);
        }
        if (mail_cache_map(cache, offset, field_hdr->size) < 0)
            return -1;
    }

    *offset_r = offset;
    return 0;
}

int mail_cache_header_fields_read(struct mail_cache *cache)
{
    const struct mail_cache_header_fields *field_hdr = NULL;
    struct mail_cache_field field;
    const uint32_t *last_used, *sizes;
    const uint8_t *types, *decisions;
    const char *p, *names, *end;
    char *orig_key;
    void *orig_value;
    unsigned int fidx, new_fields_count;
    enum mail_cache_decision_type dec;
    time_t max_drop_time;
    uint32_t offset, i;

    if (mail_cache_header_fields_get_offset(cache, &offset, TRUE) < 0)
        return -1;

    if (offset == 0) {
        /* no fields - the file is empty */
        return 0;
    }

    field_hdr = CONST_PTR_OFFSET(cache->data, offset);
    if (offset + field_hdr->size > cache->mmap_length) {
        mail_cache_set_corrupted(cache,
                     "field header points outside file");
        return -1;
    }

    /* check the fixed size of the header. name[] has to be checked
       separately */
    if (field_hdr->size < sizeof(*field_hdr) +
        field_hdr->fields_count * (sizeof(uint32_t)*2 + 1 + 2)) {
        mail_cache_set_corrupted(cache, "invalid field header size");
        return -1;
    }

    field_hdr = CONST_PTR_OFFSET(cache->data, offset);
    new_fields_count = field_hdr->fields_count;

    if (new_fields_count != 0) {
        cache->file_field_map =
            i_realloc(cache->file_field_map,
                  cache->file_fields_count *
                  sizeof(unsigned int),
                  new_fields_count * sizeof(unsigned int));
    } else {
        i_free_and_null(cache->file_field_map);
    }
    cache->file_fields_count = new_fields_count;

    last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED());
    sizes = CONST_PTR_OFFSET(field_hdr,
        MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count));
    types = CONST_PTR_OFFSET(field_hdr,
        MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count));
    decisions = CONST_PTR_OFFSET(field_hdr,
        MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count));
    names = CONST_PTR_OFFSET(field_hdr,
        MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count));
    end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);

    /* clear the old mapping */
    for (i = 0; i < cache->fields_count; i++)
        cache->field_file_map[i] = (uint32_t)-1;

    max_drop_time = cache->index->map->hdr.day_stamp == 0 ? 0 :
        cache->index->map->hdr.day_stamp - MAIL_CACHE_FIELD_DROP_SECS;

    memset(&field, 0, sizeof(field));
    for (i = 0; i < field_hdr->fields_count; i++) {
        for (p = names; p != end && *p != '\0'; p++) ;
        if (p == end || *names == '\0') {
            mail_cache_set_corrupted(cache,
                "field header names corrupted");
            return -1;
        }

        if (types[i] > MAIL_CACHE_FIELD_COUNT) {
            mail_cache_set_corrupted(cache, "field type corrupted");
            return -1;
        }
        if (!field_decision_is_valid(decisions[i])) {
            mail_cache_set_corrupted(cache,
                "field decision type corrupted");
            return -1;
        }

        if (hash_table_lookup_full(cache->field_name_hash, names,
                       &orig_key, &orig_value)) {
            /* already exists, see if decision can be updated */
            fidx = POINTER_CAST_TO(orig_value, unsigned int);
            if (!cache->fields[fidx].decision_dirty) {
                cache->fields[fidx].field.decision =
                    decisions[i];
            }
            if (field_type_verify(cache, fidx,
                          types[i], sizes[i]) < 0)
                return -1;
        } else {
            field.name = names;
            field.type = types[i];
            field.field_size = sizes[i];
            field.decision = decisions[i];
            mail_cache_register_fields(cache, &field, 1);
            fidx = field.idx;
        }
        if (cache->field_file_map[fidx] != (uint32_t)-1) {
            mail_cache_set_corrupted(cache,
                "Duplicated field in header: %s", names);
            return -1;
        }
        cache->fields[fidx].used = TRUE;

        cache->field_file_map[fidx] = i;
        cache->file_field_map[i] = fidx;

        /* update last_used if it's newer than ours */
        if ((time_t)last_used[i] > cache->fields[fidx].field.last_used)
            cache->fields[fidx].field.last_used = last_used[i];

        dec = cache->fields[fidx].field.decision;
        if (cache->fields[fidx].field.last_used < max_drop_time &&
            cache->fields[fidx].field.last_used != 0 &&
            (dec & MAIL_CACHE_DECISION_FORCED) == 0 &&
            dec != MAIL_CACHE_DECISION_NO) {
            /* time to drop this field. don't bother dropping
               fields that have never been used. */
            cache->need_compress_file_seq = cache->hdr->file_seq;
        }

                names = p + 1;
    }
    return 0;
}

static void copy_to_buf(struct mail_cache *cache, buffer_t *dest, bool add_new,
            size_t offset, size_t size)
{
    const void *data;
    unsigned int i, field;

    /* copy the existing fields */
    for (i = 0; i < cache->file_fields_count; i++) {
        field = cache->file_field_map[i];
                data = CONST_PTR_OFFSET(&cache->fields[field], offset);
        buffer_append(dest, data, size);
    }
    if (!add_new)
        return;

    /* copy newly wanted fields */
    for (i = 0; i < cache->fields_count; i++) {
        if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
            data = CONST_PTR_OFFSET(&cache->fields[i], offset);
            buffer_append(dest, data, size);
        }
    }
}

static void copy_to_buf_byte(struct mail_cache *cache, buffer_t *dest,
                 bool add_new, size_t offset)
{
    const int *data;
    unsigned int i, field;
    uint8_t byte;

    /* copy the existing fields */
    for (i = 0; i < cache->file_fields_count; i++) {
        field = cache->file_field_map[i];
                data = CONST_PTR_OFFSET(&cache->fields[field], offset);
        byte = (uint8_t)*data;
        buffer_append(dest, &byte, 1);
    }
    if (!add_new)
        return;

    /* copy newly wanted fields */
    for (i = 0; i < cache->fields_count; i++) {
        if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
            data = CONST_PTR_OFFSET(&cache->fields[i], offset);
            byte = (uint8_t)*data;
            buffer_append(dest, &byte, 1);
        }
    }
}

static int mail_cache_header_fields_update_locked(struct mail_cache *cache)
{
    buffer_t *buffer;
    uint32_t i, offset, dec_offset;
    int ret = 0;

    if (mail_cache_header_fields_read(cache) < 0 ||
        mail_cache_header_fields_get_offset(cache, &offset, FALSE) < 0)
        return -1;

    buffer = buffer_create_dynamic(pool_datastack_create(), 256);

    copy_to_buf(cache, buffer, FALSE,
            offsetof(struct mail_cache_field, last_used),
            sizeof(uint32_t));
    ret = mail_cache_write(cache, buffer->data, buffer->used,
                   offset + MAIL_CACHE_FIELD_LAST_USED());
    if (ret == 0) {
        buffer_set_used_size(buffer, 0);
        copy_to_buf_byte(cache, buffer, FALSE,
                 offsetof(struct mail_cache_field, decision));

        dec_offset = offset +
            MAIL_CACHE_FIELD_DECISION(cache->file_fields_count);
        ret = mail_cache_write(cache, buffer->data, buffer->used,
                       dec_offset);
        if (ret == 0) {
            for (i = 0; i < cache->file_fields_count; i++)
                cache->fields[i].decision_dirty = FALSE;
        }
    }

    if (ret == 0)
        cache->field_header_write_pending = FALSE;
    return ret;
}

int mail_cache_header_fields_update(struct mail_cache *cache)
{
    int ret;

    if (cache->locked) {
        T_BEGIN {
            ret = mail_cache_header_fields_update_locked(cache);
        } T_END;
        return ret;
    }

    if (mail_cache_lock(cache, FALSE) <= 0)
        return -1;

    T_BEGIN {
        ret = mail_cache_header_fields_update_locked(cache);
    } T_END;
    if (mail_cache_unlock(cache) < 0)
        ret = -1;
    return ret;
}

void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest)
{
    struct mail_cache_header_fields hdr;
    unsigned int field;
    const char *name;
    uint32_t i;

    memset(&hdr, 0, sizeof(hdr));
    hdr.fields_count = cache->file_fields_count;
    for (i = 0; i < cache->fields_count; i++) {
        if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i))
            hdr.fields_count++;
    }
    buffer_append(dest, &hdr, sizeof(hdr));

    /* we have to keep the field order for the existing fields. */
    copy_to_buf(cache, dest, TRUE,
            offsetof(struct mail_cache_field, last_used),
            sizeof(uint32_t));
    copy_to_buf(cache, dest, TRUE,
            offsetof(struct mail_cache_field, field_size),
            sizeof(uint32_t));
    copy_to_buf_byte(cache, dest, TRUE,
             offsetof(struct mail_cache_field, type));
    copy_to_buf_byte(cache, dest, TRUE,
             offsetof(struct mail_cache_field, decision));

    i_assert(dest->used == sizeof(hdr) +
         (sizeof(uint32_t)*2 + 2) * hdr.fields_count);

    /* add existing fields' names */
    for (i = 0; i < cache->file_fields_count; i++) {
        field = cache->file_field_map[i];
        name = cache->fields[field].field.name;
        buffer_append(dest, name, strlen(name)+1);
    }
    /* add newly wanted fields' names */
    for (i = 0; i < cache->fields_count; i++) {
        if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
            name = cache->fields[i].field.name;
            buffer_append(dest, name, strlen(name)+1);
        }
    }

    hdr.size = dest->used;
    buffer_write(dest, 0, &hdr, sizeof(hdr));

    if ((hdr.size & 3) != 0)
        buffer_append_zero(dest, 4 - (hdr.size & 3));
}

int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
                         uint32_t *offset_r)
{
    if (mail_cache_header_fields_get_offset(cache, offset_r, FALSE) < 0)
        return -1;

    if (*offset_r == 0) {
        *offset_r = offsetof(struct mail_cache_header,
                     field_header_offset);
    } else {
        *offset_r += offsetof(struct mail_cache_header_fields,
                      next_offset);
    }
    return 0;
}