src/lib-index/mail-cache-transaction.c

	mail-cache-transaction.c revision 8aacc9e7c84f8376822823ec98c2f551d4919b2e
bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (C) 2003-2004 Timo Sirainen */
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen#include "lib.h"
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen#include "buffer.h"
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen#include "byteorder.h"
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen#include "file-set-size.h"
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen#include "mmap-util.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "mail-cache-private.h"
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen#include <sys/stat.h>
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstruct mail_cache_transaction_ctx {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    struct mail_cache *cache;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    struct mail_cache_view *view;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    struct mail_index_transaction *trans;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    unsigned int next_unused_header_lowwater;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    struct mail_cache_record cache_rec;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    buffer_t *cache_data;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    uint32_t first_seq, last_seq, prev_seq;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    enum mail_cache_field prev_fields;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    buffer_t *cache_marks;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    uint32_t used_file_size;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen};
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainenstatic const unsigned char *null4[] = { 0, 0, 0, 0 };
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainenint mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock,
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen                 struct mail_index_transaction *t,
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen                 struct mail_cache_transaction_ctx **ctx_r)
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        struct mail_cache_transaction_ctx *ctx;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    int ret;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    i_assert(view->cache->trans_ctx == NULL);
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    ret = mail_cache_lock(view->cache, nonblock);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (ret <= 0)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return ret;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    ctx = i_new(struct mail_cache_transaction_ctx, 1);
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    ctx->cache = view->cache;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    ctx->view = view;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    ctx->trans = t;
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1);
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    ctx->used_file_size = nbo_to_uint32(ctx->cache->hdr->used_file_size);
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    view->cache->trans_ctx = ctx;
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    *ctx_r = ctx;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    return 1;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenint mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    int ret = 0;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    i_assert(ctx->cache->trans_ctx != NULL);
0de1e3762d960366b5e311cad2b80b5d66f82c5bTimo Sirainen
0de1e3762d960366b5e311cad2b80b5d66f82c5bTimo Sirainen    (void)mail_cache_transaction_rollback(ctx);
0de1e3762d960366b5e311cad2b80b5d66f82c5bTimo Sirainen
0de1e3762d960366b5e311cad2b80b5d66f82c5bTimo Sirainen    if (mail_cache_unlock(ctx->cache) < 0)
0de1e3762d960366b5e311cad2b80b5d66f82c5bTimo Sirainen        ret = -1;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    ctx->cache->trans_ctx = NULL;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    if (ctx->cache_marks != NULL)
c2fbbf7515aa419dc8b2d62a3c2bb0471d51a391Timo Sirainen        buffer_free(ctx->cache_marks);
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    buffer_free(ctx->cache_data);
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    i_free(ctx);
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    return ret;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen{
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    ctx->next_unused_header_lowwater = 0;
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    ctx->first_seq = ctx->last_seq = ctx->prev_seq = 0;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    ctx->prev_fields = 0;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (ctx->cache_marks != NULL)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        buffer_set_used_size(ctx->cache_marks, 0);
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    buffer_set_used_size(ctx->cache_data, 0);
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic void mark_update(buffer_t **buf, uint32_t offset, uint32_t data)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (*buf == NULL)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        *buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    buffer_append(*buf, &offset, sizeof(offset));
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    buffer_append(*buf, &data, sizeof(data));
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic int write_mark_updates(struct mail_cache *cache)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    const uint32_t *data, *end;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    size_t size;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    data = buffer_get_data(cache->trans_ctx->cache_marks, &size);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    end = data + size/sizeof(uint32_t);
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    while (data < end) {
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        if (pwrite(cache->fd, data+1, sizeof(*data), data[0]) < 0) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            mail_cache_set_syscall_error(cache, "pwrite()");
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen            return -1;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        }
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        data += 2;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    }
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    return 0;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainenstatic int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen{
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    struct mail_cache *cache = ctx->cache;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    uint32_t cont;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    /* write everything to disk */
4020006405eab97087f985a64b505d3cf48ceac7Timo Sirainen    if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) {
4020006405eab97087f985a64b505d3cf48ceac7Timo Sirainen        mail_cache_set_syscall_error(cache, "msync()");
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return -1;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    }
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (fdatasync(cache->fd) < 0) {
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        mail_cache_set_syscall_error(cache, "fdatasync()");
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return -1;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (ctx->cache_marks == NULL ||
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        buffer_get_used_size(ctx->cache_marks) == 0)
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen        return 0;
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    /* now that we're sure it's written, set on all the used-bits */
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    if (write_mark_updates(cache) < 0)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return -1;
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen    /* update continued records count */
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    cont = nbo_to_uint32(cache->hdr->continued_record_count);
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    cont += buffer_get_used_size(ctx->cache_marks) /
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen        (sizeof(uint32_t) * 2);
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    if (cont * 100 / cache->index->hdr->messages_count >=
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        COMPRESS_CONTINUED_PERCENTAGE &&
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        ctx->used_file_size >= COMPRESS_MIN_SIZE) {
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        /* too many continued rows, compress */
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch        cache->need_compress = TRUE;
ce763c83d556eea9b74fc63e5d41f3b71a7e72deTimo Sirainen    }
20b2d47ed762ca2c009aa37e360af6f223ac71bdTimo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    cache->hdr->continued_record_count = uint32_to_nbo(cont);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    return 0;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen}
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainenstatic int
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainenmail_cache_grow(struct mail_cache_transaction_ctx *ctx, uint32_t size)
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen{
13d58f3be3356c68ad4f53781f69bc6fe5623700Timo Sirainen        struct mail_cache *cache = ctx->cache;
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen    struct stat st;
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen    uoff_t grow_size, new_fsize;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    new_fsize = ctx->used_file_size + size;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    if (grow_size < 16384)
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        grow_size = 16384;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    new_fsize += grow_size;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    new_fsize &= ~1023;
013a0e03309a4a77552f6e8dd46d06c3d3f60f53Timo Sirainen
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    if (fstat(cache->fd, &st) < 0) {
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        mail_cache_set_syscall_error(cache, "fstat()");
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen        return -1;
4fc52b7b25c3d3f42348903e0154840f8761f306Timo Sirainen    }
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    if (ctx->used_file_size + size <= (uoff_t)st.st_size) {
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        /* no need to grow, just update mmap */
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        if (mail_cache_mmap_update(cache, 0, 0) < 0)
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen            return -1;
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        i_assert(cache->mmap_length >= (uoff_t)st.st_size);
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen        return 0;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    }
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    if (file_set_size(cache->fd, (off_t)new_fsize) < 0) {
896093ae6961612ac6ef8722d79dd457f1b43dd0Timo Sirainen        mail_cache_set_syscall_error(cache, "file_set_size()");
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        return -1;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    }
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    return mail_cache_mmap_update(cache, 0, 0);
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen}
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainenstatic uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen                    uint32_t size)
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen{
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    /* NOTE: must be done within transaction or rollback would break it */
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    uint32_t offset;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    i_assert((size & 3) == 0);
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    offset = ctx->used_file_size;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    if (offset >= 0x40000000) {
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen        mail_index_set_error(ctx->cache->index,
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen                     "Cache file too large: %s",
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen                     ctx->cache->filepath);
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen        return 0;
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen    }
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen    if (offset + size > ctx->cache->mmap_length) {
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen        if (mail_cache_grow(ctx, size) < 0)
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen            return 0;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    }
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    ctx->used_file_size += size;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    return offset;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen}
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainen
55058d5f50f7e53bb22df4fa6ae951c8001416abTimo Sirainenstatic int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen{
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen    struct mail_cache *cache = ctx->cache;
1176124297af5c56e932c0863c6637ff21d8a0efTimo Sirainen    struct mail_cache_record *cache_rec, *next;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    const struct mail_index_record *rec;
13d58f3be3356c68ad4f53781f69bc6fe5623700Timo Sirainen    struct mail_index_map *map;
13d58f3be3356c68ad4f53781f69bc6fe5623700Timo Sirainen    uint32_t write_offset, update_offset;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    const void *buf;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    size_t size, buf_size;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    int ret;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    buf = buffer_get_data(ctx->cache_data, &buf_size);
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    size = sizeof(*cache_rec) + buf_size;
8d35582f2577c64517b2341c5d6477c7010e0a0cPhil Carmody    ctx->cache_rec.size = uint32_to_nbo(size);
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    ret = mail_index_lookup_full(ctx->view->view, ctx->prev_seq,
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen                     &map, &rec);
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    if (ret < 0)
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen        return -1;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    if (map->hdr->cache_file_seq != cache->hdr->file_seq) {
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen        /* FIXME: we should check if newer file is available? */
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen        ret = 0;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    }
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    if (ret == 0) {
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen        /* it's been expunged already, do nothing */
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen    } else {
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen        write_offset = mail_cache_append_space(ctx, size);
0a1ec0f2a38370e8a073a60023b4365491947f6fTimo Sirainen        if (write_offset == 0)
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen            return -1;
d798962a54c5cda054d57a0cfc7e5f47dfa20f6eTimo Sirainen
        cache_rec = mail_cache_get_record(cache, rec->cache_offset,
                          TRUE);
        if (cache_rec == NULL) {
            /* first cache record - update offset in index file */
            mail_index_update_cache(ctx->trans, ctx->prev_seq,
                        write_offset);
        } else {
            /* find the last cache record */
            while ((next = mail_cache_get_next_record(cache,
                                  cache_rec)) != NULL)
                cache_rec = next;

            /* mark next_offset to be updated later */
            update_offset = (char *) &cache_rec->next_offset -
                (char *) cache->mmap_base;
            mark_update(&ctx->cache_marks, update_offset,
                    mail_cache_uint32_to_offset(write_offset));
        }

        memcpy((char *) cache->mmap_base + write_offset,
               &ctx->cache_rec, sizeof(ctx->cache_rec));
        memcpy((char *) cache->mmap_base + write_offset +
               sizeof(ctx->cache_rec), buf, buf_size);
    }

    /* reset the write context */
    ctx->prev_seq = 0;
    ctx->prev_fields = 0;

    memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
    buffer_set_used_size(ctx->cache_data, 0);
    return 0;
}

int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
{
    int ret = 0;

    if (ctx->cache->disabled) {
        mail_cache_transaction_flush(ctx);
        return 0;
    }

    if (ctx->prev_seq != 0) {
        if (mail_cache_write(ctx) < 0)
            return -1;
    }

    ctx->cache->hdr->used_file_size = uint32_to_nbo(ctx->used_file_size);

    if (commit_all_changes(ctx) < 0)
        ret = -1;

    if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
        /* they're all used - compress the cache to get more */
        ctx->cache->need_compress = TRUE;
    }

    mail_cache_transaction_flush(ctx);
    return ret;
}

void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache *cache = ctx->cache;
    unsigned int i;

    /* no need to actually modify the file - we just didn't update
       used_file_size */
    ctx->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);

    /* make sure we don't cache the headers */
    for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
        uint32_t offset = cache->hdr->header_offsets[i];
        if (mail_cache_offset_to_uint32(offset) == 0)
            cache->split_offsets[i] = 1;
    }

    mail_cache_transaction_flush(ctx);
}

static const char *write_header_string(const char *const headers[],
                       uint32_t *size_r)
{
    buffer_t *buffer;
    size_t size;

    buffer = buffer_create_dynamic(pool_datastack_create(),
                       512, (size_t)-1);

    while (*headers != NULL) {
        if (buffer_get_used_size(buffer) != 0)
            buffer_append(buffer, "\n", 1);
        buffer_append(buffer, *headers, strlen(*headers));
        headers++;
    }
    buffer_append(buffer, null4, 1);

    size = buffer_get_used_size(buffer);
    if ((size & 3) != 0) {
        buffer_append(buffer, null4, 4 - (size & 3));
        size += 4 - (size & 3);
    }
    *size_r = size;
    return buffer_get_data(buffer, NULL);
}

int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
                 unsigned int idx, const char *const headers[])
{
    struct mail_cache *cache = ctx->cache;
    uint32_t offset, update_offset, size;
    const char *header_str, *prev_str;

    i_assert(*headers != NULL);
    i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
    i_assert(idx >= ctx->next_unused_header_lowwater);
    i_assert(mail_cache_offset_to_uint32(cache->hdr->
                         header_offsets[idx]) == 0);

    t_push();

    header_str = write_header_string(headers, &size);
    if (idx != 0) {
        prev_str = mail_cache_get_header_fields_str(cache, idx-1);
        if (prev_str == NULL) {
            t_pop();
            return FALSE;
        }

        i_assert(strcmp(header_str, prev_str) != 0);
    }

    offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
    if (offset != 0) {
        memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
               header_str, size);

        size = uint32_to_nbo(size);
        memcpy((char *) cache->mmap_base + offset,
               &size, sizeof(uint32_t));

        /* update cached headers */
        cache->split_offsets[idx] = cache->hdr->header_offsets[idx];
        cache->split_headers[idx] =
            mail_cache_split_header(cache, header_str);

        /* mark used-bit to be updated later. not really needed for
           read-safety, but if transaction get rolled back we can't let
           this point to invalid location. */
        update_offset = (char *) &cache->hdr->header_offsets[idx] -
            (char *) cache->mmap_base;
        mark_update(&ctx->cache_marks, update_offset,
                mail_cache_uint32_to_offset(offset));

        /* make sure get_header_fields() still works for this header
           while the transaction isn't yet committed. */
        ctx->next_unused_header_lowwater = idx + 1;
    }

    t_pop();
    return offset > 0;
}

static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
                enum mail_cache_field field)
{
    const unsigned char *buf;
    unsigned int mask;
    uint32_t data_size;
    size_t offset = 0;
    int i;

    buf = buffer_get_data(ctx->cache_data, NULL);

    for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
        if ((field & mask) != 0)
            return offset;

        if ((ctx->cache_rec.fields & mask) != 0) {
            if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
                data_size = mail_cache_field_sizes[i];
            else {
                memcpy(&data_size, buf + offset,
                       sizeof(data_size));
                data_size = nbo_to_uint32(data_size);
                offset += sizeof(data_size);
            }
            offset += (data_size + 3) & ~3;
        }
    }

    i_unreached();
    return offset;
}

static int get_field_num(enum mail_cache_field field)
{
    unsigned int mask;
    int i;

    for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
        if ((field & mask) != 0)
            return i;
    }

    return -1;
}

int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
           enum mail_cache_field field,
           const void *data, size_t data_size)
{
    uint32_t nb_data_size;
    size_t full_size, offset;
    unsigned char *buf;
    int field_num;

    i_assert(data_size > 0);
    i_assert(data_size < (uint32_t)-1);

    nb_data_size = uint32_to_nbo((uint32_t)data_size);

    if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
        field_num = get_field_num(field);
        i_assert(field_num != -1);
        i_assert(mail_cache_field_sizes[field_num] == data_size);
    } else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
        i_assert(((char *) data)[data_size-1] == '\0');
    }

    if (ctx->prev_seq != seq && ctx->prev_seq != 0) {
        if (mail_cache_write(ctx) < 0)
            return -1;
    }
    ctx->prev_seq = seq;

    i_assert((ctx->cache_rec.fields & field) == 0);

    full_size = (data_size + 3) & ~3;
    if ((field & MAIL_CACHE_FIXED_MASK) == 0)
        full_size += sizeof(nb_data_size);

    /* fields must be ordered. find where to insert it. */
    if (field > ctx->cache_rec.fields)
                buf = buffer_append_space_unsafe(ctx->cache_data, full_size);
    else {
        offset = get_insert_offset(ctx, field);
        buffer_copy(ctx->cache_data, offset + full_size,
                ctx->cache_data, offset, (size_t)-1);
        buf = buffer_get_space_unsafe(ctx->cache_data,
                          offset, full_size);
    }
    ctx->cache_rec.fields |= field;

    /* @UNSAFE */
    if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
        memcpy(buf, &nb_data_size, sizeof(nb_data_size));
        buf += sizeof(nb_data_size);
    }
    memcpy(buf, data, data_size); buf += data_size;
    if ((data_size & 3) != 0)
        memset(buf, 0, 4 - (data_size & 3));

    /* remember the transaction sequence range */
    if (seq < ctx->first_seq || ctx->first_seq == 0)
        ctx->first_seq = seq;
    if (seq > ctx->last_seq)
        ctx->last_seq = seq;
    ctx->prev_fields |= field;

    return 0;
}

int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq)
{
    struct mail_cache *cache = ctx->cache;
    struct mail_cache_record *cache_rec;
    uint32_t deleted_space;
    uoff_t max_del_space;

    cache_rec = mail_cache_lookup(ctx->view, seq, 0);
    if (cache_rec == NULL)
        return 0;

    /* we'll only update the deleted_space in header. we can't really
       do any actual deleting as other processes might still be using
       the data. also it's actually useful as some index views are still
       able to ask cached data from messages that have already been
       expunged. */
    deleted_space = nbo_to_uint32(cache->hdr->deleted_space);

    do {
        deleted_space -= nbo_to_uint32(cache_rec->size);
        cache_rec = mail_cache_get_next_record(cache, cache_rec);
    } while (cache_rec != NULL);

    /* see if we've reached the max. deleted space in file */
    max_del_space = ctx->used_file_size / 100 * COMPRESS_PERCENTAGE;
    if (deleted_space >= max_del_space &&
        ctx->used_file_size >= COMPRESS_MIN_SIZE)
        cache->need_compress = TRUE;

    cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
    return 0;
}

int
mail_cache_transaction_autocommit(struct mail_cache_view *view,
                  uint32_t seq, enum mail_cache_field fields)
{
    struct mail_cache *cache = view->cache;

    if (cache->trans_ctx != NULL &&
        cache->trans_ctx->first_seq <= seq &&
        cache->trans_ctx->last_seq >= seq &&
        (cache->trans_ctx->prev_seq != seq || fields == 0 ||
         (cache->trans_ctx->prev_fields & fields) != 0)) {
        /* write non-index changes */
        if (cache->trans_ctx->prev_seq == seq) {
            if (mail_cache_write(cache->trans_ctx) < 0)
                return -1;
        }

        if (mail_cache_transaction_commit(cache->trans_ctx) < 0)
            return -1;
    }

    return 0;
}

int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
                   enum mail_cache_record_flag flags)
{
    return -1;
}