mail-cache-transaction.c revision fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic const unsigned char *null4[] = { 0, 0, 0, 0 };
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx = i_new(struct mail_cache_transaction_ctx, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1);
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen ctx->used_file_size = nbo_to_uint32(ctx->cache->hdr->used_file_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->first_seq = ctx->last_seq = ctx->prev_seq = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void mark_update(buffer_t **buf, uint32_t offset, uint32_t data)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen *buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int write_mark_updates(struct mail_cache *cache)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen data = buffer_get_data(cache->trans_ctx->cache_marks, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (pwrite(cache->fd, data+1, sizeof(*data), data[0]) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_cache_set_syscall_error(cache, "pwrite()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* write everything to disk */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_cache_set_syscall_error(cache, "msync()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_cache_set_syscall_error(cache, "fdatasync()");
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* now that we're sure it's written, set on all the used-bits */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* update continued records count */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cont = nbo_to_uint32(cache->hdr->continued_record_count);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cont += buffer_get_used_size(ctx->cache_marks) /
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (cont * 100 / cache->index->hdr->messages_count >=
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* too many continued rows, compress */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cache->hdr->continued_record_count = uint32_to_nbo(cont);
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainenmail_cache_grow(struct mail_cache_transaction_ctx *ctx, uint32_t size)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_cache_set_syscall_error(cache, "fstat()");
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen if (ctx->used_file_size + size <= (uoff_t)st.st_size) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* no need to grow, just update mmap */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (mail_cache_map(cache, 0, (size_t)st.st_size) < 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(cache->mmap_length >= (uoff_t)st.st_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (file_set_size(cache->fd, (off_t)new_fsize) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_cache_set_syscall_error(cache, "file_set_size()");
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return mail_cache_map(cache, 0, (size_t)new_fsize);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* NOTE: must be done within transaction or rollback would break it */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "Cache file too large: %s",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (offset + size > ctx->cache->mmap_length) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const void *buf;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buf = buffer_get_data(ctx->cache_data, &buf_size);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen ret = mail_cache_lookup_offset(ctx->view, ctx->prev_seq, &offset, TRUE);
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen /* it's been expunged already, do nothing */
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen write_offset = mail_cache_append_space(ctx, size);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen cache_rec = mail_cache_get_record(cache, offset, TRUE);
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen /* first cache record - update offset in index file */
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen mail_index_update_cache(ctx->trans, ctx->prev_seq,
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen /* find offset to last cache record */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen cache_rec = mail_cache_get_record(cache, offset,
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen /* mark next_offset to be updated later */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen offset = mail_cache_offset_to_uint32(offset) +
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen offsetof(struct mail_cache_record, next_offset);
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen memcpy((char *) cache->mmap_base + write_offset,
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen memcpy((char *) cache->mmap_base + write_offset +
da5d50534cfca45d0aaaf0bdac17b287b4588809Timo Sirainen /* reset the write context */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen ctx->cache->hdr->used_file_size = uint32_to_nbo(ctx->used_file_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* they're all used - compress the cache to get more */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenvoid mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int i;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* no need to actually modify the file - we just didn't update
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen used_file_size */
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen ctx->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* make sure we don't cache the headers */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t offset = cache->hdr->header_offsets[i];
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic const char *write_header_string(const char *const headers[],
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer = buffer_create_dynamic(pool_datastack_create(),
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append(buffer, *headers, strlen(*headers));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int idx, const char *const headers[])
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(idx >= ctx->next_unused_header_lowwater);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(mail_cache_offset_to_uint32(cache->hdr->
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen header_str = write_header_string(headers, &size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen prev_str = mail_cache_get_header_fields_str(cache, idx-1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* update cached headers */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cache->split_offsets[idx] = cache->hdr->header_offsets[idx];
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* mark used-bit to be updated later. not really needed for
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen read-safety, but if transaction get rolled back we can't let
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen this point to invalid location. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen update_offset = (char *) &cache->hdr->header_offsets[idx] -
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* make sure get_header_fields() still works for this header
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen while the transaction isn't yet committed. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const unsigned char *buf;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int mask;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int get_field_num(enum mail_cache_field field)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int mask;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned char *buf;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen nb_data_size = uint32_to_nbo((uint32_t)data_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(mail_cache_field_sizes[field_num] == data_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert(((char *) data)[data_size-1] == '\0');
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->prev_seq != seq && ctx->prev_seq != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_assert((ctx->cache_rec.fields & field) == 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* fields must be ordered. find where to insert it. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buf = buffer_append_space_unsafe(ctx->cache_data, full_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_copy(ctx->cache_data, offset + full_size,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buf = buffer_get_space_unsafe(ctx->cache_data,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* @UNSAFE */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memcpy(buf, &nb_data_size, sizeof(nb_data_size));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memcpy(buf, data, data_size); buf += data_size;
ba482d3624ca4f1b3d638e6e8470ba5134f21493Timo Sirainen /* remember the transaction sequence range */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (seq < ctx->first_seq || ctx->first_seq == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cache_rec = mail_cache_lookup(ctx->view, seq, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* we'll only update the deleted_space in header. we can't really
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen do any actual deleting as other processes might still be using
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen the data. also it's actually useful as some index views are still
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen able to ask cached data from messages that have already been
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen deleted_space = nbo_to_uint32(cache->hdr->deleted_space);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen deleted_space -= nbo_to_uint32(cache_rec->size);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen cache_rec = mail_cache_get_record(cache, cache_rec->next_offset,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* see if we've reached the max. deleted space in file */
56f45b3f3ae20e5c933701f4657dda5ef1916855Timo Sirainen max_del_space = ctx->used_file_size / 100 * COMPRESS_PERCENTAGE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_cache_transaction_autocommit(struct mail_cache_view *view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (cache->trans_ctx->prev_seq != seq || fields == 0 ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (cache->trans_ctx->prev_fields & fields) != 0)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* write non-index changes */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_cache_transaction_commit(cache->trans_ctx) < 0)