bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "lib.h"
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen#include "array.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "ostream.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "nfs-workarounds.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "read-full.h"
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen#include "file-dotlock.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "file-cache.h"
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen#include "file-set-size.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "mail-cache-private.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen#include <stdio.h>
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen#include <sys/stat.h>
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstruct mail_cache_copy_context {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen struct mail_cache *cache;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_t *buffer, *field_seen;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(unsigned int) bitmask_pos;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen uint32_t *field_file_map;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen uint8_t field_seen_value;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen bool new_msg;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainenstruct mail_cache_compress_lock {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct dotlock *dotlock;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen};
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainenstatic void
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainenmail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen const struct mail_cache_iterate_field *field)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen{
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen unsigned char *dest;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen unsigned int i, *pos;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen if (*pos == 0) {
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen /* we decided to drop this field */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen return;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen }
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen for (i = 0; i < field->size; i++)
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen dest[i] |= ((const unsigned char*)field->data)[i];
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen}
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenstatic void
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenmail_cache_compress_field(struct mail_cache_copy_context *ctx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen const struct mail_cache_iterate_field *field)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen struct mail_cache_field *cache_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen enum mail_cache_decision_type dec;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen uint32_t file_field_idx, size32;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen uint8_t *field_seen;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen file_field_idx = ctx->field_file_map[field->field_idx];
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen if (file_field_idx == (uint32_t)-1)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen return;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache_field = &ctx->cache->fields[field->field_idx].field;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen field_seen = buffer_get_space_unsafe(ctx->field_seen,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen field->field_idx, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (*field_seen == ctx->field_seen_value) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* duplicate */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen mail_cache_merge_bitmask(ctx, field);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen return;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen *field_seen = ctx->field_seen_value;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen if (ctx->new_msg) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (dec == MAIL_CACHE_DECISION_NO)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen return;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (dec != MAIL_CACHE_DECISION_YES)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen return;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen }
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8ae72ad7d0c69e972cfa65d1e2ce4e3e9a8b765cTimo Sirainen if (cache_field->field_size == UINT_MAX) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen size32 = (uint32_t)field->size;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx->buffer, &size32, sizeof(size32));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen /* remember the position in case we need to update it */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen unsigned int pos = ctx->buffer->used;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen buffer_append(ctx->buffer, field->data, field->size);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen if ((field->size & 3) != 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainenstatic uint32_t get_next_file_seq(struct mail_cache *cache)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen const struct mail_index_ext *ext;
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen struct mail_index_view *view;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen uint32_t file_seq;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen /* make sure we look up the latest reset_id */
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen if (mail_index_refresh(cache->index) < 0)
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen return -1;
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen view = mail_index_view_open(cache->index);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen ext = mail_index_view_get_ext(view, cache->ext_id);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen file_seq = cache->hdr->file_seq + 1;
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen mail_index_view_close(&view);
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return file_seq != 0 ? file_seq : 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen}
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainenstatic void
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainenmail_cache_compress_get_fields(struct mail_cache_copy_context *ctx,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen unsigned int used_fields_count)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen{
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen struct mail_cache *cache = ctx->cache;
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen struct mail_cache_field *field;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen unsigned int i, j, idx;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen /* Make mail_cache_header_fields_get() return the fields in
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen the same order as we saved them. */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen memcpy(cache->field_file_map, ctx->field_file_map,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen sizeof(uint32_t) * cache->fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen /* reverse mapping */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_fields_count = used_fields_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_free(cache->file_field_map);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_field_map = used_fields_count == 0 ? NULL :
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_new(unsigned int, used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen for (i = j = 0; i < cache->fields_count; i++) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen idx = cache->field_file_map[i];
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen if (idx != (uint32_t)-1) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_assert(idx < used_fields_count &&
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen cache->file_field_map != NULL &&
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_field_map[idx] == 0);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->file_field_map[idx] = i;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen j++;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen }
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen /* change permanent decisions to temporary decisions.
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen if they're still permanent they'll get updated later. */
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen field = &cache->fields[i].field;
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen if (field->decision == MAIL_CACHE_DECISION_YES)
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen field->decision = MAIL_CACHE_DECISION_TEMP;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_assert(j == used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen buffer_set_used_size(ctx->buffer, 0);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen mail_cache_header_fields_get(cache, ctx->buffer);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen}
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainenstatic int
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainenmail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
f883b315ca72073b58020798e6d907340b327228Timo Sirainen int fd, uint32_t *file_seq_r, uoff_t *file_size_r, uint32_t *max_uid_r,
c740209eb3db031022c6fd436fa2ef1aa0ec46b8Timo Sirainen ARRAY_TYPE(uint32_t) *ext_offsets)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_cache_copy_context ctx;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen struct mail_cache_lookup_iterate_ctx iter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen struct mail_cache_iterate_field field;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen struct mail_index_view *view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_cache_view *cache_view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen const struct mail_index_header *idx_hdr;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_cache_header hdr;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_cache_record cache_rec;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct ostream *output;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen uint32_t message_count, seq, first_new_seq, ext_offset;
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen unsigned int i, used_fields_count, orig_fields_count, record_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen time_t max_drop_time;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
f883b315ca72073b58020798e6d907340b327228Timo Sirainen *max_uid_r = 0;
f883b315ca72073b58020798e6d907340b327228Timo Sirainen
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen /* get the latest info on fields */
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen if (mail_cache_header_fields_read(cache) < 0)
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen return -1;
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen view = mail_index_transaction_get_view(trans);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen cache_view = mail_cache_view_open(cache, view);
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen output = o_stream_create_fd_file(fd, 0, FALSE);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&hdr);
920b9f0fdfa5a5d7763e05736601a31bcb291a53Timo Sirainen hdr.major_version = MAIL_CACHE_MAJOR_VERSION;
920b9f0fdfa5a5d7763e05736601a31bcb291a53Timo Sirainen hdr.minor_version = MAIL_CACHE_MINOR_VERSION;
811f2e26d9782d9cb99fdf82e18ffa0a77564fe2Timo Sirainen hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen hdr.indexid = cache->index->indexid;
cfd0e35fce37786f1bb186386ebe29aeef7dbf31Timo Sirainen hdr.file_seq = get_next_file_seq(cache);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&ctx);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen ctx.cache = cache;
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen ctx.buffer = buffer_create_dynamic(default_pool, 4096);
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen ctx.field_seen = buffer_create_dynamic(default_pool, 64);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx.field_seen_value = 0;
b879ed8dd4b5850987e6b89a92f794d87c6be7d7Timo Sirainen ctx.field_file_map = t_new(uint32_t, cache->fields_count + 1);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen t_array_init(&ctx.bitmask_pos, 32);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen /* @UNSAFE: drop unused fields and create a field mapping for
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen used fields */
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen idx_hdr = mail_index_get_header(view);
8d77ab87a57ae98532c62e5d1548bd131dee3074Timo Sirainen max_drop_time = idx_hdr->day_stamp == 0 ? 0 :
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen idx_hdr->day_stamp -
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen cache->index->optimization_set.cache.unaccessed_field_drop_secs;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen orig_fields_count = cache->fields_count;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen if (cache->file_fields_count == 0) {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen /* creating the initial cache file. add all fields. */
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen for (i = 0; i < orig_fields_count; i++)
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen ctx.field_file_map[i] = i;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen used_fields_count = i;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen } else {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen for (i = used_fields_count = 0; i < orig_fields_count; i++) {
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen struct mail_cache_field_private *priv =
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen &cache->fields[i];
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen enum mail_cache_decision_type dec =
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen priv->field.decision;
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen /* if the decision isn't forced and this field hasn't
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen been accessed for a while, drop it */
cc3ccdab8a510d88fecedf95187465bf84833711Timo Sirainen if ((dec & MAIL_CACHE_DECISION_FORCED) == 0 &&
008a83e9f680f04f69789fb702232416eab2a86cTimo Sirainen priv->field.last_used < max_drop_time &&
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen !priv->adding) {
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen dec = MAIL_CACHE_DECISION_NO;
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen priv->field.decision = dec;
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen }
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen /* drop all fields we don't want */
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen if ((dec & ~MAIL_CACHE_DECISION_FORCED) ==
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen MAIL_CACHE_DECISION_NO && !priv->adding) {
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen priv->used = FALSE;
008a83e9f680f04f69789fb702232416eab2a86cTimo Sirainen priv->field.last_used = 0;
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen }
ddbad7a661c0663fafd2b79393efa85f840d6af6Timo Sirainen
b8d232d88018c5cafd2f3be5a181d318137a45f2Timo Sirainen ctx.field_file_map[i] = !priv->used ?
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen (uint32_t)-1 : used_fields_count++;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen /* get sequence of first message which doesn't need its temp fields
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen removed. */
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen first_new_seq = mail_cache_get_first_new_seq(view);
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen message_count = mail_index_view_get_messages_count(view);
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen i_array_init(ext_offsets, message_count); record_count = 0;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen for (seq = 1; seq <= message_count; seq++) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen if (mail_index_transaction_is_expunged(trans, seq)) {
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(ext_offsets);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen continue;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen }
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen ctx.new_msg = seq >= first_new_seq;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_set_used_size(ctx.buffer, 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (++ctx.field_seen_value == 0) {
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen 0, buffer_get_size(ctx.field_seen));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx.field_seen_value++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&cache_rec);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen mail_cache_lookup_iter_init(cache_view, seq, &iter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen while (mail_cache_lookup_iter_next(&iter, &field) > 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen mail_cache_compress_field(&ctx, &field);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
330d54e9e3bf076efe91f101c562d1268160b128Timo Sirainen if (ctx.buffer->used == sizeof(cache_rec) ||
67bbcd664bebce9a507a49c67273be4814d07c97Timo Sirainen ctx.buffer->used > cache->index->optimization_set.cache.record_max_size) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen /* nothing cached */
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen ext_offset = 0;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen } else {
f883b315ca72073b58020798e6d907340b327228Timo Sirainen mail_index_lookup_uid(view, seq, max_uid_r);
330d54e9e3bf076efe91f101c562d1268160b128Timo Sirainen cache_rec.size = ctx.buffer->used;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen ext_offset = output->offset;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen buffer_write(ctx.buffer, 0, &cache_rec,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen sizeof(cache_rec));
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, ctx.buffer->data, cache_rec.size);
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen record_count++;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen }
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen array_append(ext_offsets, &ext_offset, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen i_assert(orig_fields_count == cache->fields_count);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
f0ff961282e618945dfe997dc45ff95d656e5790Timo Sirainen hdr.record_count = record_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen mail_cache_compress_get_fields(&ctx, used_fields_count);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
289064eb21595d3e4460439eccdc48232d13f5e1Timo Sirainen hdr.backwards_compat_used_file_size = output->offset;
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&ctx.buffer);
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen buffer_free(&ctx.field_seen);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)o_stream_seek(output, 0);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
128ea07dab8d67124ea74bcc085a478784b6358aTimo Sirainen mail_cache_view_close(&cache_view);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
ad9afb64630511d5e25bc5bc11c5304986156928Timo Sirainen if (o_stream_finish(output) < 0) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_cache_set_syscall_error(cache, "write()");
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen o_stream_destroy(&output);
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen array_free(ext_offsets);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
f883b315ca72073b58020798e6d907340b327228Timo Sirainen *file_size_r = output->offset;
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen o_stream_destroy(&output);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
b780aa272b742a43579cdb523cc79cc8d4521306Timo Sirainen if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen if (fdatasync(fd) < 0) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen mail_cache_set_syscall_error(cache, "fdatasync()");
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen array_free(ext_offsets);
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen return -1;
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen }
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen *file_seq_r = hdr.file_seq;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainenstatic int
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainenmail_cache_compress_write(struct mail_cache *cache,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct mail_index_transaction *trans,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen int fd, const char *temp_path, bool *unlock)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen{
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct stat st;
f883b315ca72073b58020798e6d907340b327228Timo Sirainen uint32_t file_seq, old_offset, max_uid;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen ARRAY_TYPE(uint32_t) ext_offsets;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen const uint32_t *offsets;
f883b315ca72073b58020798e6d907340b327228Timo Sirainen uoff_t file_size;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen unsigned int i, count;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
f883b315ca72073b58020798e6d907340b327228Timo Sirainen if (mail_cache_copy(cache, trans, fd, &file_seq, &file_size,
f883b315ca72073b58020798e6d907340b327228Timo Sirainen &max_uid, &ext_offsets) < 0)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return -1;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (fstat(fd, &st) < 0) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen mail_cache_set_syscall_error(cache, "fstat()");
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen array_free(&ext_offsets);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return -1;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (rename(temp_path, cache->filepath) < 0) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen mail_cache_set_syscall_error(cache, "rename()");
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen array_free(&ext_offsets);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return -1;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
f883b315ca72073b58020798e6d907340b327228Timo Sirainen if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_DEBUG) != 0) {
f883b315ca72073b58020798e6d907340b327228Timo Sirainen i_debug("%s: Compressed, file_seq changed %u -> %u, "
f883b315ca72073b58020798e6d907340b327228Timo Sirainen "size=%"PRIuUOFF_T", max_uid=%u", cache->filepath,
f883b315ca72073b58020798e6d907340b327228Timo Sirainen cache->need_compress_file_seq, file_seq,
f883b315ca72073b58020798e6d907340b327228Timo Sirainen file_size, max_uid);
f883b315ca72073b58020798e6d907340b327228Timo Sirainen }
f883b315ca72073b58020798e6d907340b327228Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* once we're sure that the compression was successful,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen update the offsets */
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen offsets = array_get(&ext_offsets, &count);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen for (i = 0; i < count; i++) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (offsets[i] != 0) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen mail_index_update_ext(trans, i + 1, cache->ext_id,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen &offsets[i], &old_offset);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen array_free(&ext_offsets);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (*unlock) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen (void)mail_cache_unlock(cache);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen *unlock = FALSE;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen mail_cache_file_close(cache);
f330b7d9e14255fc06bc82908d9bc5a12cccb424Timo Sirainen cache->opened = TRUE;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen cache->fd = fd;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen cache->st_ino = st.st_ino;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen cache->st_dev = st.st_dev;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen cache->field_header_write_pending = FALSE;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return 0;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen}
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainenstatic int mail_cache_compress_has_file_changed(struct mail_cache *cache)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen{
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen struct mail_cache_header hdr;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen unsigned int i;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen int fd, ret;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen for (i = 0;; i++) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen fd = nfs_safe_open(cache->filepath, O_RDONLY);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (fd == -1) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (errno == ENOENT)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_cache_set_syscall_error(cache, "open()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen ret = read_full(fd, &hdr, sizeof(hdr));
4307c886579381dbb1897ea1388ae6978c96f560Timo Sirainen i_close_fd(&fd);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret >= 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret == 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 0;
c18e2301f4c81f2819711661890db7796d4450c3Timo Sirainen if (cache->need_compress_file_seq == 0) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* previously it didn't exist or it
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen was unusable and was just unlinked */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
f0339f522dc9c8e2e8a29ef9a3f937c431c6bd1bTimo Sirainen return hdr.file_seq != cache->need_compress_file_seq ? 1 : 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_cache_set_syscall_error(cache, "read()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen}
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainenstatic int mail_cache_compress_dotlock(struct mail_cache *cache,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct dotlock **dotlock_r)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen{
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (file_dotlock_create(&cache->dotlock_settings, cache->filepath,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen DOTLOCK_CREATE_FLAG_NONBLOCK, dotlock_r) <= 0) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (errno != EAGAIN)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen mail_cache_set_syscall_error(cache, "file_dotlock_open()");
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return -1;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return 0;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen}
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenstatic int mail_cache_compress_locked(struct mail_cache *cache,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen struct mail_index_transaction *trans,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen bool *unlock, struct dotlock **dotlock_r)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen const char *temp_path;
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen const void *data;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen int fd, ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* There are two possible locking situations here:
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen a) Cache is locked against any modifications.
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen b) Cache doesn't exist or is unusable. There's no lock.
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen Because the cache lock itself is unreliable, we'll be using a
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen separate dotlock to guard against two processes compressing the
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen cache at the same time. */
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (mail_cache_compress_dotlock(cache, dotlock_r) < 0)
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen return -1;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* we've locked the cache compression now. if somebody else had just
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen recreated the cache, reopen the cache and return success. */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret < 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen /* was just compressed, forget this */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->need_compress_file_seq = 0;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen file_dotlock_delete(dotlock_r);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen return mail_cache_reopen(cache) < 0 ? -1 : 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
5b47087754f4008e88c93386d524e07db535c132Timo Sirainen if (cache->fd != -1) {
5b47087754f4008e88c93386d524e07db535c132Timo Sirainen /* make sure we have mapped it before reading. */
5b47087754f4008e88c93386d524e07db535c132Timo Sirainen if (mail_cache_map(cache, 0, 0, &data) < 0)
5b47087754f4008e88c93386d524e07db535c132Timo Sirainen return -1;
5b47087754f4008e88c93386d524e07db535c132Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* we want to recreate the cache. write it first to a temporary file */
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (fd == -1)
f1700e6d755306a2b137fad88ea5bf8b9fc6ec4fTimo Sirainen return -1;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (mail_cache_compress_write(cache, trans, fd, temp_path, unlock) < 0) {
4307c886579381dbb1897ea1388ae6978c96f560Timo Sirainen i_close_fd(&fd);
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen i_unlink(temp_path);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (cache->file_cache != NULL)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen file_cache_set_fd(cache->file_cache, cache->fd);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
82ed69779f49bd71ef1b570ce8aca67d357dbee8Timo Sirainen if (mail_cache_map(cache, 0, 0, &data) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_header_fields_read(cache) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->need_compress_file_seq = 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen}
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainenint mail_cache_compress(struct mail_cache *cache,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct mail_index_transaction *trans,
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct mail_cache_compress_lock **lock_r)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen{
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct dotlock *dotlock = NULL;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen bool unlock = FALSE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen int ret;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen i_assert(!cache->compressing);
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen *lock_r = NULL;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen *lock_r = i_new(struct mail_cache_compress_lock, 1);
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen return 0;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen /* compression isn't very efficient with small read()s */
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen if (cache->map_with_read) {
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen cache->map_with_read = FALSE;
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen if (cache->read_buf != NULL)
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen buffer_set_used_size(cache->read_buf, 0);
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen cache->hdr = NULL;
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen cache->mmap_length = 0;
de9b2ee7878a73346ba0eee34798abb22ffcfcb6Timo Sirainen }
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen /* we're using dotlocking, cache file creation itself creates
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen the dotlock file we need. */
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen if (!MAIL_CACHE_IS_UNUSABLE(cache)) {
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen mail_index_flush_read_cache(cache->index,
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen cache->filepath, cache->fd,
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen FALSE);
bc564f1d3d953cf724828322b11ae89e0f59ffc9Timo Sirainen }
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen } else {
d0bbbc7057aa33b52ee378196dee7d773437468fTimo Sirainen switch (mail_cache_try_lock(cache)) {
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen case -1:
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* already locked or some other error */
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen return -1;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen case 0:
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* cache is broken or doesn't exist.
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen just start creating it. */
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen break;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen default:
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen /* locking succeeded. */
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen unlock = TRUE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen }
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen cache->compressing = TRUE;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen ret = mail_cache_compress_locked(cache, trans, &unlock, &dotlock);
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen cache->compressing = FALSE;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen if (unlock) {
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen if (mail_cache_unlock(cache) < 0)
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen ret = -1;
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen }
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (ret < 0) {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (dotlock != NULL)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen file_dotlock_delete(&dotlock);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen /* the fields may have been updated in memory already.
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen reverse those changes by re-reading them from file. */
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen (void)mail_cache_header_fields_read(cache);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen } else {
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen *lock_r = i_new(struct mail_cache_compress_lock, 1);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen (*lock_r)->dotlock = dotlock;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen }
5724e7103eed12fe36b55a7b5a8653284a2184b9Timo Sirainen return ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainenvoid mail_cache_compress_unlock(struct mail_cache_compress_lock **_lock)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen{
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen struct mail_cache_compress_lock *lock = *_lock;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen *_lock = NULL;
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen if (lock->dotlock != NULL)
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen file_dotlock_delete(&lock->dotlock);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen i_free(lock);
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen}
0f833d18af36bf307cd69f235eb3fb779245b478Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenbool mail_cache_need_compress(struct mail_cache *cache)
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen{
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen return cache->need_compress_file_seq != 0 &&
18d92dbbb752c79dc461514e52f7ef11847e636bTimo Sirainen (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) == 0 &&
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen !cache->index->readonly;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen}