mail-cache-compress.c revision 369a1084c500a9df7448ffa9409ce32e42060bc2
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "lib.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "buffer.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "ostream.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "nfs-workarounds.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "read-full.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "close-keep-errno.h"
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen#include "file-dotlock.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "file-cache.h"
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen#include "file-set-size.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "mail-cache-private.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen#include <sys/stat.h>
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstruct mail_cache_copy_context {
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen bool new_msg;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_t *buffer, *field_seen;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen uint8_t field_seen_value;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainenstatic void mail_cache_merge_bitmask(struct mail_cache *cache, buffer_t *buffer,
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen uint32_t field, const void *data,
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen size_t data_size)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen{
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen void *buf_data;
72e9e7ad158101d46860b42c4080e894485c78c3Timo Sirainen uint32_t buf_field;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen unsigned int i, buf_data_size;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen size_t pos, buf_size;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen buf_data = buffer_get_modifiable_data(buffer, &buf_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen for (pos = sizeof(struct mail_cache_record); pos < buf_size; ) {
72e9e7ad158101d46860b42c4080e894485c78c3Timo Sirainen buf_field = *((uint32_t *)PTR_OFFSET(buf_data, pos));
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen pos += sizeof(uint32_t);
4823da41d112ff9f5e8f088b0e60d1636e01ff92Timo Sirainen i_assert(buf_field < cache->fields_count);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
afa6ac39d1d6df246d4e7352288c2a0388276a24Timo Sirainen buf_data_size = cache->fields[buf_field].field.field_size;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen if (buf_data_size == (unsigned int)-1) {
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen buf_data_size =
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen *((uint32_t *)PTR_OFFSET(buf_data, pos));
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen pos += sizeof(uint32_t);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen }
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
72e9e7ad158101d46860b42c4080e894485c78c3Timo Sirainen if (buf_field == field) {
4823da41d112ff9f5e8f088b0e60d1636e01ff92Timo Sirainen /* @UNSAFE: found it, do the merging */
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen unsigned char *dest = PTR_OFFSET(buf_data, pos);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen i_assert(buf_data_size == data_size);
4823da41d112ff9f5e8f088b0e60d1636e01ff92Timo Sirainen i_assert(pos + buf_data_size <= buf_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen for (i = 0; i < buf_data_size; i++)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen dest[i] |= ((const unsigned char*)data)[i];
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen break;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen }
9945ad9e528188521d876b80f08a648072ffa207Timo Sirainen pos += (buf_data_size + 3) & ~3;
4823da41d112ff9f5e8f088b0e60d1636e01ff92Timo Sirainen i_assert(pos <= buf_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen }
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen}
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstatic int
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainenmail_cache_compress_callback(struct mail_cache_view *view, uint32_t field,
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen const void *data, size_t data_size, void *context)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct mail_cache_copy_context *ctx = context;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen struct mail_cache_field *cache_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen enum mail_cache_decision_type dec;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen uint8_t *field_seen;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen uint32_t size32;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen cache_field = &view->cache->fields[field].field;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen field_seen = buffer_get_space_unsafe(ctx->field_seen, field, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (*field_seen == ctx->field_seen_value) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* duplicate */
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen mail_cache_merge_bitmask(view->cache, ctx->buffer,
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen field, data, data_size);
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return 1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen *field_seen = ctx->field_seen_value;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen if (ctx->new_msg) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (dec == MAIL_CACHE_DECISION_NO)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen return 1;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (dec != MAIL_CACHE_DECISION_YES)
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen return 1;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen }
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen buffer_append(ctx->buffer, &field, sizeof(field));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen if (cache_field->field_size == (unsigned int)-1) {
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen size32 = (uint32_t)data_size;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx->buffer, &size32, sizeof(size32));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx->buffer, data, data_size);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen if ((data_size & 3) != 0)
d5ac54ef50db16b50689b5c8b7bb64d344190832Timo Sirainen buffer_append_zero(ctx->buffer, 4 - (data_size & 3));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen return 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenstatic uint32_t
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenget_next_file_seq(struct mail_cache *cache, struct mail_index_view *view)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen const struct mail_index_ext *ext;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen uint32_t file_seq;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen ext = mail_index_view_get_ext(view, cache->ext_id);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen return file_seq != 0 ? file_seq : 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen}
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainenstatic int
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainenmail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_cache_copy_context ctx;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_cache_view *cache_view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_index_transaction *t;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen const struct mail_index_header *idx_hdr;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct mail_cache_header hdr;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_cache_record cache_rec;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen struct ostream *output;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen buffer_t *buffer;
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen uint32_t i, message_count, seq, first_new_seq, old_offset;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen uoff_t offset;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
325d4ad220bd13f6d176391d962a0e33c856a7f6Timo Sirainen /* get sequence of first message which doesn't need its temp fields
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen removed. */
b20fb5b1df9d604a7541f5118fc5b4b466d211efTimo Sirainen idx_hdr = mail_index_get_header(view);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (idx_hdr->day_first_uid[7] == 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen first_new_seq = 1;
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen message_count = mail_index_view_get_messages_count(view);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen } else {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (mail_index_lookup_uid_range(view, idx_hdr->day_first_uid[7],
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen (uint32_t)-1, &first_new_seq,
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen &message_count) < 0)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (first_new_seq == 0)
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen first_new_seq = message_count+1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen cache_view = mail_cache_view_open(cache, view);
7c95b03620a03a43dd72d39608cea5fc77393ad6Timo Sirainen t = mail_index_transaction_begin(view, FALSE, TRUE);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen output = o_stream_create_file(fd, default_pool, 0, FALSE);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen memset(&hdr, 0, sizeof(hdr));
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen hdr.version = MAIL_CACHE_VERSION;
811f2e26d9782d9cb99fdf82e18ffa0a77564fe2Timo Sirainen hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen hdr.indexid = idx_hdr->indexid;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen hdr.file_seq = get_next_file_seq(cache, view);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen o_stream_send(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen memset(&ctx, 0, sizeof(ctx));
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen ctx.buffer = buffer_create_dynamic(default_pool, 4096);
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen ctx.field_seen = buffer_create_dynamic(default_pool, 64);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx.field_seen_value = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen mail_index_ext_reset(t, cache->ext_id, hdr.file_seq);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen for (seq = 1; seq <= message_count; seq++) {
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen ctx.new_msg = seq >= first_new_seq;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_set_used_size(ctx.buffer, 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (++ctx.field_seen_value == 0) {
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen 0, buffer_get_size(ctx.field_seen));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen ctx.field_seen_value++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen memset(&cache_rec, 0, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen (void)mail_cache_foreach(cache_view, seq,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_cache_compress_callback, &ctx);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen cache_rec.size = buffer_get_used_size(ctx.buffer);
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen if (cache_rec.size == sizeof(cache_rec))
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen continue;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen mail_index_update_ext(t, seq, cache->ext_id, &output->offset,
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen &old_offset);
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen buffer_write(ctx.buffer, 0, &cache_rec, sizeof(cache_rec));
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen o_stream_send(output, ctx.buffer->data, cache_rec.size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen if (cache->fields_count != 0) {
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen hdr.field_header_offset =
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen mail_index_uint32_to_offset(output->offset);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
7797aa2479e99aeb71057b7a2584b2cb72e4d3f8Timo Sirainen /* we wrote everything using our internal field ids. so we want
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen mail_cache_header_fields_get() to use them and ignore any
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen existing id mappings in the old cache file. */
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen cache->file_fields_count = 0;
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen for (i = 0; i < cache->fields_count; i++)
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen cache->field_file_map[i] = (uint32_t)-1;
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen t_push();
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen buffer = buffer_create_dynamic(pool_datastack_create(), 256);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen mail_cache_header_fields_get(cache, buffer);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen o_stream_send(output, buffer_get_data(buffer, NULL),
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen buffer_get_used_size(buffer));
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen t_pop();
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen }
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen hdr.used_file_size = output->offset;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen buffer_free(ctx.buffer);
7416b94f38e82381abd9cee660efdcf3e7b773afTimo Sirainen buffer_free(ctx.field_seen);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen o_stream_seek(output, 0);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen o_stream_send(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen mail_cache_view_close(cache_view);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen if (o_stream_flush(output) < 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen errno = output->stream_errno;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen mail_cache_set_syscall_error(cache, "o_stream_flush()");
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen (void)mail_index_transaction_rollback(&t);
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen o_stream_destroy(&output);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen /* grow the file some more. doesn't matter if it fails */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen o_stream_destroy(&output);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen if (!cache->index->fsync_disable) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen if (fdatasync(fd) < 0) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen mail_cache_set_syscall_error(cache, "fdatasync()");
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen (void)mail_index_transaction_rollback(&t);
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen return -1;
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen }
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen return mail_index_transaction_commit(&t, &seq, &offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainenstatic int mail_cache_compress_has_file_changed(struct mail_cache *cache)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen{
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen struct mail_cache_header hdr;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen unsigned int i;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen int fd, ret;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen for (i = 0;; i++) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen fd = nfs_safe_open(cache->filepath, O_RDONLY);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (fd == -1) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (errno == ENOENT)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_cache_set_syscall_error(cache, "open()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen ret = read_full(fd, &hdr, sizeof(hdr));
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen close_keep_errno(fd);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret >= 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret == 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (cache->need_compress_file_seq == (uint32_t)-1) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen /* previously it didn't exist */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return 1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return hdr.file_seq != cache->need_compress_file_seq;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen mail_cache_set_syscall_error(cache, "read()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen}
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenstatic int mail_cache_compress_locked(struct mail_cache *cache,
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen struct mail_index_view *view,
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen bool *unlock)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen struct dotlock *dotlock;
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen mode_t old_mask;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen int fd, ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* get the latest info on fields */
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_header_fields_read(cache) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen old_mask = umask(cache->index->mode ^ 0666);
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen fd = file_dotlock_open(&cache->dotlock_settings, cache->filepath,
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen 0, &dotlock);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen umask(old_mask);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (fd == -1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_cache_set_syscall_error(cache, "file_dotlock_open()");
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen if (ret < 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen /* was just compressed, forget this */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->need_compress_file_seq = 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen file_dotlock_delete(&dotlock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return mail_cache_reopen(cache);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#ifdef DEBUG
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen i_warning("Compressing cache file %s (%u)",
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->filepath, cache->need_compress_file_seq);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#endif
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen if (cache->index->gid != (gid_t)-1 &&
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen fchown(fd, (uid_t)-1, cache->index->gid) < 0) {
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen mail_cache_set_syscall_error(cache, "fchown()");
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen file_dotlock_delete(&dotlock);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen return -1;
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen }
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen if (mail_cache_copy(cache, view, fd) < 0) {
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen (void)file_dotlock_delete(&dotlock);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen if (file_dotlock_replace(&dotlock,
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen mail_cache_set_syscall_error(cache,
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen "file_dotlock_replace()");
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen (void)close(fd);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen mail_cache_file_close(cache);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen cache->fd = fd;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (cache->file_cache != NULL)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen file_cache_set_fd(cache->file_cache, cache->fd);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_map(cache, 0, 0) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen if (mail_cache_header_fields_read(cache) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen cache->need_compress_file_seq = 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen}
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenint mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen{
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen bool unlock = FALSE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen int ret;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen return 0;
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen /* we're using dotlocking, cache file creation itself creates
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen the dotlock file we need. */
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen return mail_cache_compress_locked(cache, view, &unlock);
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen }
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen switch (mail_cache_lock(cache)) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen case -1:
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen case 0:
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen /* couldn't lock, either it's broken or doesn't exist.
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen just start creating it. */
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen return mail_cache_compress_locked(cache, view, &unlock);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen default:
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen /* locking succeeded. */
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen unlock = TRUE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen ret = mail_cache_compress_locked(cache, view, &unlock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen if (mail_cache_unlock(cache) < 0)
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen ret = -1;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen return ret;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenbool mail_cache_need_compress(struct mail_cache *cache)
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen{
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen return cache->need_compress_file_seq != 0;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen}