mail-cache-compress.c revision e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "lib.h"
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen#include "array.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "ostream.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "nfs-workarounds.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "read-full.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "close-keep-errno.h"
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen#include "file-dotlock.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "file-cache.h"
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen#include "file-set-size.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "mail-cache-private.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen#include <sys/stat.h>
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstruct mail_cache_copy_context {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen struct mail_cache *cache;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_t *buffer, *field_seen;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen ARRAY_DEFINE(bitmask_pos, unsigned int);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen uint32_t *field_file_map;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen uint8_t field_seen_value;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen bool new_msg;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainenstatic void
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainenmail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen const struct mail_cache_iterate_field *field)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen{
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen unsigned char *dest;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen unsigned int i, *pos;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen if (*pos == 0) {
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen /* we decided to drop this field */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen return;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen }
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen for (i = 0; i < field->size; i++)
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen dest[i] |= ((const unsigned char*)field->data)[i];
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen}
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenstatic void
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenmail_cache_compress_field(struct mail_cache_copy_context *ctx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen const struct mail_cache_iterate_field *field)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen struct mail_cache_field *cache_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen enum mail_cache_decision_type dec;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen uint32_t file_field_idx, size32;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen uint8_t *field_seen;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen file_field_idx = ctx->field_file_map[field->field_idx];
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen if (file_field_idx == (uint32_t)-1)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen return;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache_field = &ctx->cache->fields[field->field_idx].field;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen field_seen = buffer_get_space_unsafe(ctx->field_seen,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen field->field_idx, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (*field_seen == ctx->field_seen_value) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* duplicate */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen mail_cache_merge_bitmask(ctx, field);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen return;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen *field_seen = ctx->field_seen_value;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen if (ctx->new_msg) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (dec == MAIL_CACHE_DECISION_NO)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen return;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (dec != MAIL_CACHE_DECISION_YES)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen return;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen }
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen if (cache_field->field_size == (unsigned int)-1) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen size32 = (uint32_t)field->size;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx->buffer, &size32, sizeof(size32));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen /* remember the position in case we need to update it */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen unsigned int pos = ctx->buffer->used;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen buffer_append(ctx->buffer, field->data, field->size);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen if ((field->size & 3) != 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenstatic uint32_t
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenget_next_file_seq(struct mail_cache *cache, struct mail_index_view *view)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen const struct mail_index_ext *ext;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen uint32_t file_seq;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen ext = mail_index_view_get_ext(view, cache->ext_id);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen file_seq = cache->hdr->file_seq + 1;
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return file_seq != 0 ? file_seq : 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen}
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainenstatic void
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainenmail_cache_compress_get_fields(struct mail_cache_copy_context *ctx,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen unsigned int used_fields_count)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen{
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen struct mail_cache *cache = ctx->cache;
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen struct mail_cache_field *field;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen unsigned int i, j, idx;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen /* Make mail_cache_header_fields_get() return the fields in
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen the same order as we saved them. */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen memcpy(cache->field_file_map, ctx->field_file_map,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen sizeof(uint32_t) * cache->fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen /* reverse mapping */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_fields_count = used_fields_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_free(cache->file_field_map);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_field_map = used_fields_count == 0 ? NULL :
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_new(unsigned int, used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen for (i = j = 0; i < cache->fields_count; i++) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen idx = cache->field_file_map[i];
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen if (idx != (uint32_t)-1) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_assert(idx < used_fields_count &&
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen cache->file_field_map != NULL &&
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_field_map[idx] == 0);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_field_map[idx] = i;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen j++;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen }
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen /* change permanent decisions to temporary decisions.
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen if they're still permanent they'll get updated later. */
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen field = &cache->fields[i].field;
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen if (field->decision == MAIL_CACHE_DECISION_YES)
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen field->decision = MAIL_CACHE_DECISION_TEMP;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_assert(j == used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen buffer_set_used_size(ctx->buffer, 0);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen mail_cache_header_fields_get(cache, ctx->buffer);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen}
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainenstatic int
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainenmail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen int fd, uint32_t *file_seq_r,
c740209eb3db031022c6fd436fa2ef1aa0ec46b8Timo Sirainen ARRAY_TYPE(uint32_t) *ext_offsets)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_cache_copy_context ctx;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen struct mail_cache_lookup_iterate_ctx iter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen struct mail_cache_iterate_field field;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen struct mail_index_view *view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_cache_view *cache_view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen const struct mail_index_header *idx_hdr;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_cache_header hdr;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_cache_record cache_rec;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct ostream *output;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen uint32_t message_count, seq, first_new_seq, ext_offset;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen unsigned int i, used_fields_count, orig_fields_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen time_t max_drop_time;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen view = mail_index_transaction_get_view(trans);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen cache_view = mail_cache_view_open(cache, view);
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen output = o_stream_create_fd_file(fd, 0, FALSE);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen memset(&hdr, 0, sizeof(hdr));
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen hdr.version = MAIL_CACHE_VERSION;
811f2e26d9782d9cb99fdf82e18ffa0a77564fe2Timo Sirainen hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen hdr.indexid = cache->index->indexid;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen hdr.file_seq = get_next_file_seq(cache, view);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen memset(&ctx, 0, sizeof(ctx));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen ctx.cache = cache;
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen ctx.buffer = buffer_create_dynamic(default_pool, 4096);
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen ctx.field_seen = buffer_create_dynamic(default_pool, 64);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx.field_seen_value = 0;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen ctx.field_file_map = t_new(uint32_t, cache->fields_count + 1);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen t_array_init(&ctx.bitmask_pos, 32);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen /* @UNSAFE: drop unused fields and create a field mapping for
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen used fields */
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen idx_hdr = mail_index_get_header(view);
8d77ab87a57ae98532c62e5d1548bd131dee3074Timo Sirainen max_drop_time = idx_hdr->day_stamp == 0 ? 0 :
8d77ab87a57ae98532c62e5d1548bd131dee3074Timo Sirainen idx_hdr->day_stamp - MAIL_CACHE_FIELD_DROP_SECS;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen orig_fields_count = cache->fields_count;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen if (cache->file_fields_count == 0) {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen /* creating the initial cache file. add all fields. */
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen for (i = 0; i < orig_fields_count; i++)
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen ctx.field_file_map[i] = i;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen used_fields_count = i;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen } else {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen for (i = used_fields_count = 0; i < orig_fields_count; i++) {
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen struct mail_cache_field_private *priv =
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen &cache->fields[i];
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen enum mail_cache_decision_type dec =
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen priv->field.decision;
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen /* if the decision isn't forced and this field hasn't
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen been accessed for a while, drop it */
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen if ((dec & MAIL_CACHE_DECISION_FORCED) == 0 &&
008a83e9f680f04f69789fb702232416eab2a86cTimo Sirainen priv->field.last_used < max_drop_time &&
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen !priv->adding) {
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen dec = MAIL_CACHE_DECISION_NO;
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen priv->field.decision = dec;
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen }
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen /* drop all fields we don't want */
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen if ((dec & ~MAIL_CACHE_DECISION_FORCED) ==
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen MAIL_CACHE_DECISION_NO && !priv->adding) {
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen priv->used = FALSE;
008a83e9f680f04f69789fb702232416eab2a86cTimo Sirainen priv->field.last_used = 0;
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen }
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen ctx.field_file_map[i] = !priv->used ?
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen (uint32_t)-1 : used_fields_count++;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen /* get sequence of first message which doesn't need its temp fields
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen removed. */
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen first_new_seq = mail_cache_get_first_new_seq(view);
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen message_count = mail_index_view_get_messages_count(view);
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen i_array_init(ext_offsets, message_count);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen for (seq = 1; seq <= message_count; seq++) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen if (mail_index_transaction_is_expunged(trans, seq)) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen (void)array_append_space(ext_offsets);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen continue;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen }
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen ctx.new_msg = seq >= first_new_seq;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_set_used_size(ctx.buffer, 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (++ctx.field_seen_value == 0) {
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen 0, buffer_get_size(ctx.field_seen));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx.field_seen_value++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen memset(&cache_rec, 0, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen mail_cache_lookup_iter_init(cache_view, seq, &iter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen while (mail_cache_lookup_iter_next(&iter, &field) > 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen mail_cache_compress_field(&ctx, &field);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen cache_rec.size = buffer_get_used_size(ctx.buffer);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen if (cache_rec.size == sizeof(cache_rec)) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen /* nothing cached */
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen ext_offset = 0;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen } else {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen ext_offset = output->offset;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen buffer_write(ctx.buffer, 0, &cache_rec,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen sizeof(cache_rec));
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, ctx.buffer->data, cache_rec.size);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen }
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen array_append(ext_offsets, &ext_offset, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_assert(orig_fields_count == cache->fields_count);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen mail_cache_compress_get_fields(&ctx, used_fields_count);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen hdr.used_file_size = output->offset;
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&ctx.buffer);
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&ctx.field_seen);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen o_stream_seek(output, 0);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen mail_cache_view_close(&cache_view);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (o_stream_nfinish(output) < 0) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_cache_set_syscall_error(cache, "write()");
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen o_stream_destroy(&output);
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen array_free(ext_offsets);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen /* grow the file some more. doesn't matter if it fails */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen o_stream_destroy(&output);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen if (fdatasync(fd) < 0) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen mail_cache_set_syscall_error(cache, "fdatasync()");
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen array_free(ext_offsets);
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen return -1;
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen }
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen *file_seq_r = hdr.file_seq;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainenstatic int mail_cache_compress_has_file_changed(struct mail_cache *cache)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen{
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen struct mail_cache_header hdr;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen unsigned int i;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen int fd, ret;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen for (i = 0;; i++) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen fd = nfs_safe_open(cache->filepath, O_RDONLY);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (fd == -1) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (errno == ENOENT)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_cache_set_syscall_error(cache, "open()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen ret = read_full(fd, &hdr, sizeof(hdr));
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen close_keep_errno(fd);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret >= 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret == 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 0;
c18e2301f4c81f2819711661890db7796d4450c3Timo Sirainen if (cache->need_compress_file_seq == 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen /* previously it didn't exist */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return hdr.file_seq != cache->need_compress_file_seq;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_cache_set_syscall_error(cache, "read()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen}
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenstatic int mail_cache_compress_locked(struct mail_cache *cache,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen struct mail_index_transaction *trans,
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen bool *unlock)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen struct dotlock *dotlock;
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen struct stat st;
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen mode_t old_mask;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen uint32_t file_seq, old_offset;
c740209eb3db031022c6fd436fa2ef1aa0ec46b8Timo Sirainen ARRAY_TYPE(uint32_t) ext_offsets;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen const uint32_t *offsets;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen unsigned int i, count;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen int fd, ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* get the latest info on fields */
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_header_fields_read(cache) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen old_mask = umask(cache->index->mode ^ 0666);
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen fd = file_dotlock_open(&cache->dotlock_settings, cache->filepath,
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen DOTLOCK_CREATE_FLAG_NONBLOCK, &dotlock);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen umask(old_mask);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (fd == -1) {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen if (errno != EAGAIN)
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen mail_cache_set_syscall_error(cache, "file_dotlock_open()");
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret < 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen /* was just compressed, forget this */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->need_compress_file_seq = 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen file_dotlock_delete(&dotlock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return mail_cache_reopen(cache);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
1f80b32fc28f7a723ff07c1694230a090808b506Timo Sirainen mail_index_fchown(cache->index, fd,
1f80b32fc28f7a723ff07c1694230a090808b506Timo Sirainen file_dotlock_get_lock_path(dotlock));
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets) < 0) {
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen /* the fields may have been updated in memory already.
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen reverse those changes by re-reading them from file. */
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen if (mail_cache_header_fields_read(cache) < 0)
25c22e54d1071d120641e9eecd0023e7373e65ffTimo Sirainen return -1;
6cb2c6ecddcdbeac9e6c73a292244747e12a793eTimo Sirainen file_dotlock_delete(&dotlock);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen if (fstat(fd, &st) < 0) {
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen mail_cache_set_syscall_error(cache, "fstat()");
6cb2c6ecddcdbeac9e6c73a292244747e12a793eTimo Sirainen file_dotlock_delete(&dotlock);
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen return -1;
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen }
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen if (file_dotlock_replace(&dotlock,
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen mail_cache_set_syscall_error(cache,
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen "file_dotlock_replace()");
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen (void)close(fd);
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen array_free(&ext_offsets);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen /* once we're sure that the compression was successful,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen update the offsets */
6e07b4251bf6a3cf34019c351a32a65c08392e58Timo Sirainen mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen offsets = array_get(&ext_offsets, &count);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen for (i = 0; i < count; i++) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen if (offsets[i] != 0) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen mail_index_update_ext(trans, i + 1, cache->ext_id,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen &offsets[i], &old_offset);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen }
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen }
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen array_free(&ext_offsets);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen mail_cache_file_close(cache);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen cache->fd = fd;
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen cache->st_ino = st.st_ino;
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen cache->st_dev = st.st_dev;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen cache->field_header_write_pending = FALSE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (cache->file_cache != NULL)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen file_cache_set_fd(cache->file_cache, cache->fd);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_map(cache, 0, 0) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_header_fields_read(cache) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->need_compress_file_seq = 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen}
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainenint mail_cache_compress(struct mail_cache *cache,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen struct mail_index_transaction *trans)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen{
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen bool unlock = FALSE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen int ret;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen i_assert(!cache->compressing);
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen return 0;
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen /* we're using dotlocking, cache file creation itself creates
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen the dotlock file we need. */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen if (!MAIL_CACHE_IS_UNUSABLE(cache)) {
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen mail_index_flush_read_cache(cache->index,
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen cache->filepath, cache->fd,
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen FALSE);
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen }
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen } else {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen switch (mail_cache_try_lock(cache)) {
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen case -1:
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen return -1;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen case 0:
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen /* couldn't lock, either it's broken or doesn't exist.
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen just start creating it. */
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen break;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen default:
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen /* locking succeeded. */
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen unlock = TRUE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen }
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen cache->compressing = TRUE;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen ret = mail_cache_compress_locked(cache, trans, &unlock);
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen cache->compressing = FALSE;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen if (unlock) {
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen if (mail_cache_unlock(cache) < 0)
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen ret = -1;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen }
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenbool mail_cache_need_compress(struct mail_cache *cache)
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen{
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen return cache->need_compress_file_seq != 0 &&
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen !cache->index->readonly;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen}