mbox-sync.c revision dda2c506c8fc8ac2f88272de4523ded42baa0aa0
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen/*
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen required disk I/O. We may need to:
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - Write missing X-UID and X-IMAPbase headers
abf015c9682f0f723db87a7c97bc284ef814818fTimo Sirainen - Write missing or broken Content-Length header if there's space
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen - Expunge specified messages
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen Here's how we do it:
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - Start reading the mails mail headers from the beginning
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - X-Keywords and X-UID headers may contain extra spaces at the end of them,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen remember how much extra each message has and offset to beginning of the
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen spaces
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - If message flags are dirty and there's enough space to write them, do it
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - If we didn't have enough space, remember how much was missing and keep
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen the total amount of them
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - When we encounter expunged message, check if the amount of empty space in
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen previous messages plus size of expunged message is enough to cover the
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen missing space. If yes,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - execute the rewrite plan
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen - forget all the messages before the expunged message. only remember
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen how much data we still have to move to cover the expunged message
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen - If we encounter end of file, grow the file and execute the rewrite plan
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen Rewrite plan goes:
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen - Start from the first message that needs more space
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen - If there's expunged messages before us, we have to write over them.
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen - Move all messages after it backwards to fill it
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen - Each moved message's X-Keywords header should have n bytes extra
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen space, unless there's not enough space to do it.
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen - If there's no expunged messages, we can move data either forward or
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen backward to get it. Calculate which requires less moving. Forward
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen counting may encounter more messages which require extra space, count
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen that too.
fa2433aebcf3fccfa30ca9eed9b1a9166cf92ee2Timo Sirainen - If we decide to move forwards and we had to go through dirty
fa2433aebcf3fccfa30ca9eed9b1a9166cf92ee2Timo Sirainen messages, do the moving from last to first dirty message
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen - If we encounter end of file, grow the file enough to get the required
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen amount of space plus enough space to fill X-Keywords headers full of
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen spaces.
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen*/
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "lib.h"
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen#include "ioloop.h"
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include "buffer.h"
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen#include "istream.h"
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen#include "file-set-size.h"
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include "str.h"
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include "write-full.h"
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen#include "istream-raw-mbox.h"
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include "mbox-storage.h"
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include "mbox-file.h"
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include "mbox-lock.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen#include "mbox-sync-private.h"
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen#include <stddef.h>
2eb2cf8eeb763bd5ca9b6848dce32f0303e88ec1Timo Sirainen#include <sys/stat.h>
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct mbox_sync_mail_context *mail_ctx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uoff_t grow_size)
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uoff_t src_offset, file_size;
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_assert(grow_size > 0);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* put the extra space between last message's header and body */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen src_offset = mail_ctx->body_offset;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_ctx->body_offset += grow_size;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen file_size - mail_ctx->body_offset) < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen istream_raw_mbox_flush(sync_ctx->input);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return 0;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen}
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct mail_index_sync_rec *sync;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen size_t size, src, dest;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen size /= sizeof(*sync);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen for (src = dest = 0; src < size; src++) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (sync[src].uid2 >= uid) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (src != dest)
b34fdb68d376d85b4880da4a4bdf67ae726a381bTimo Sirainen sync[dest] = sync[src];
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen dest++;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen}
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen /* set input->eof */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (sync_ctx->input->eof)
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen return 0;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mail_ctx->sync_ctx = sync_ctx;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_ctx->seq = ++sync_ctx->seq;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_ctx->header = sync_ctx->header;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_ctx->from_offset =
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_ctx->mail.offset =
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (mail_ctx->seq > 1 && sync_ctx->first_uid == mail_ctx->mail.uid) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* First message was expunged and this is the next one.
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen Skip \n header */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_ctx->from_offset++;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen }
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen mail_ctx->mail.body_size =
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen mail_ctx->content_length);
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen /* save the offset permanently with recent flag state */
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen mail_ctx->mail.from_offset = mail_ctx->from_offset;
542e28b384a6b26695f3e8de38fd5727d06f3333Timo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen /* need to add 'O' flag to Status-header */
714d59900e44e6e2dd744bd7b76862e1d11a4f61Timo Sirainen mail_ctx->need_rewrite = TRUE;
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen // FIXME: save it somewhere
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen }
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen return 1;
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen}
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen uint32_t uid, int *sync_expunge_r)
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen{
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen int ret;
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen *sync_expunge_r = FALSE;
18c209a06941ef583b08b173dadfbe4571995bf9Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (sync_ctx->ibox->readonly)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return 0;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen while (uid >= sync_rec->uid1) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (sync_rec->uid1 != 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_assert(uid <= sync_rec->uid2);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen buffer_append(sync_ctx->syncs, sync_rec,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sizeof(*sync_rec));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen *sync_expunge_r = TRUE;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen if (ret < 0)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (ret == 0) {
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen break;
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen }
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen }
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return 0;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen}
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen keywords_mask_t keywords)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen{
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen const struct mail_index_sync_rec *sync;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen size_t size, i;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen size /= sizeof(*sync);
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen for (i = 0; i < size; i++)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_sync_flags_apply(&sync[i], flags, keywords);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen}
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen const struct mail_index_record *rec = NULL;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uint32_t messages_count;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen while (sync_ctx->idx_seq < messages_count) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (mail_index_lookup(sync_ctx->sync_view,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen ++sync_ctx->idx_seq, &rec) < 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (uid <= rec->uid)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen break;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* externally expunged message, remove from index */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen rec = NULL;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (rec != NULL && rec->uid != uid) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen "mbox sync: UID inserted in the middle of mailbox "
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen "(%u > %u)", rec->uid, uid);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_mark_corrupted(sync_ctx->ibox->index);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen *rec_r = rec;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return 0;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen}
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct mbox_sync_mail *mail,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen int nocheck)
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen{
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen const void *data;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen uint64_t offset;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (!nocheck) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* see if from_offset needs updating */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (mail_index_lookup_extra(sync_ctx->sync_view,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync_ctx->idx_seq,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync_ctx->ibox->mbox_extra_idx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen &data) < 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen offset = *((const uint64_t *)data);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (offset == mail->from_offset)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return 0;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen } else {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen offset = mail->from_offset;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync_ctx->ibox->mbox_extra_idx, &offset);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return 0;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen}
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen struct mbox_sync_mail *mail,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen const struct mail_index_record *rec)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen keywords_mask_t idx_keywords;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uint8_t idx_flags, mbox_flags;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
9aceb071780a949f4e8bf41d3cf80735d9ac7fdfTimo Sirainen if (rec == NULL) {
9aceb071780a949f4e8bf41d3cf80735d9ac7fdfTimo Sirainen /* new message */
9aceb071780a949f4e8bf41d3cf80735d9ac7fdfTimo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
9aceb071780a949f4e8bf41d3cf80735d9ac7fdfTimo Sirainen mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen MODIFY_REPLACE, mbox_flags,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail->keywords);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen } else {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* see if flags changed */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen idx_flags = rec->flags;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mbox_sync_apply_index_syncs(sync_ctx->syncs,
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen &idx_flags, idx_keywords);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen (mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (idx_flags != mbox_flags ||
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen memcmp(idx_keywords, mail->keywords,
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen INDEX_KEYWORDS_BYTE_COUNT) != 0) {
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen MODIFY_REPLACE, mbox_flags,
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen mail->keywords);
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen }
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen }
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen rewriting would just move it anyway. */
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (sync_ctx->need_space_seq == 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen }
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen return 0;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen}
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen{
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen struct istream *input = ctx->sync_ctx->file_input;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen const unsigned char *data;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen size_t size, from_line_size;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen from_line_size = ctx->hdr_offset - ctx->from_offset;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen i_stream_seek(input, ctx->from_offset);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen for (;;) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen data = i_stream_get_data(input, &size);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (size >= from_line_size)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen size = from_line_size;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen i_stream_skip(input, size);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen from_line_size -= size;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (from_line_size == 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen break;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (i_stream_read(input) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk return 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen}
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainenstatic int
263946a65b625fd4198619a8626db0f36bbafd66Timo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
263946a65b625fd4198619a8626db0f36bbafd66Timo Sirainen{
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen string_t *str = ctx->sync_ctx->from_line;
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
02a6291366caff79793db35d479e2a062bec2af4Timo Sirainen ctx->from_offset + move_diff) < 0) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen return -1;
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen }
573085b4b25b0bbae8d27969df2c91702eefa23eTimo Sirainen
0ee3fdb5e94ae6f34cb873ca3c9858342621e55fTimo Sirainen istream_raw_mbox_flush(ctx->sync_ctx->input);
0ee3fdb5e94ae6f34cb873ca3c9858342621e55fTimo Sirainen return 0;
0ee3fdb5e94ae6f34cb873ca3c9858342621e55fTimo Sirainen}
0ee3fdb5e94ae6f34cb873ca3c9858342621e55fTimo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenstatic void
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenupdate_from_offsets(struct index_mailbox *ibox,
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk struct mail_index_transaction *t, buffer_t *mails_buf,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uint32_t seq1, uint32_t seq2)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen const struct mbox_sync_mail *mails;
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen uint32_t extra_idx = ibox->mbox_extra_idx;
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen uint64_t offset;
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen mails = buffer_get_modifyable_data(mails_buf, NULL);
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen for (; seq1 <= seq2; seq1++, mails++) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen if (mails->uid != 0) {
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen offset = mails->from_offset;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_update_extra_rec(t, seq1, extra_idx,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen &offset);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen}
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainenstatic int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen{
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (mail_ctx->sync_ctx->ibox->mbox_lock_type == F_RDLCK)
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen return -2;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen mail_ctx->mail.offset = mail_ctx->from_offset;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen mail_ctx->mail.space =
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen mail_ctx->body_offset - mail_ctx->from_offset +
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen mail_ctx->mail.body_size;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen mail_ctx->mail.body_size = 0;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (mail_ctx->sync_ctx->seq == 1) {
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen /* expunging first message, fix space to contain next
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen message's \n header too since it will be removed. */
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen mail_ctx->mail.space++;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen }
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return 0;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen}
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen{
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen off_t move_diff;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen int ret;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (sync_ctx->first_uid == 0)
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen sync_ctx->first_uid = mail_ctx->mail.uid;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (sync_ctx->ibox->readonly)
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return 0;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen /* move the header backwards to fill expunged space */
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (sync_ctx->ibox->mbox_lock_type == F_RDLCK)
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen return -2;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen move_diff = -sync_ctx->expunged_space;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen /* read the From-line before rewriting overwrites it */
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if (mbox_read_from_line(mail_ctx) < 0)
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return -1;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return -1;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (ret > 0) {
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen /* rewrite successful, write From-line to
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen new location */
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen mail_ctx->mail.from_offset += move_diff;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen mail_ctx->mail.offset += move_diff;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen if (mbox_write_from_line(mail_ctx, move_diff) < 0)
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return -1;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen }
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen } else if (mail_ctx->need_rewrite ||
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen buffer_get_used_size(sync_ctx->syncs) != 0) {
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen if (sync_ctx->ibox->mbox_lock_type == F_RDLCK)
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return -2;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
e0740628f6ca05f4bc79a9d8a90b650f4d38d4d0Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen return -1;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen } else {
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen /* nothing to do */
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen return 0;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen }
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen /* first mail with no space to write it */
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen sync_ctx->need_space_seq = sync_ctx->seq;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen sync_ctx->space_diff = 0;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen if (sync_ctx->expunged_space > 0) {
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen /* create dummy message to describe the expunged data */
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen struct mbox_sync_mail mail;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen memset(&mail, 0, sizeof(mail));
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen mail.offset = mail_ctx->from_offset -
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen sync_ctx->expunged_space;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen mail.space = sync_ctx->expunged_space;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen sync_ctx->need_space_seq--;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen buffer_append(sync_ctx->mails, &mail, sizeof(mail));
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen }
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen }
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen return 0;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen}
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainenstatic int
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen{
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen uoff_t extra_space;
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->space_diff += mail_ctx->mail.space;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (sync_ctx->space_diff < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* we have enough space now */
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen extra_space = MBOX_HEADER_EXTRA_SPACE *
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mail_ctx->mail.uid == 0 &&
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen (uoff_t)sync_ctx->space_diff > extra_space) {
c3a636e4c9ae776e0eed06b6d7ad1ccfb6003afdTimo Sirainen /* don't waste too much on extra spacing */
f934b271c69c7b3e5e3bca23ff9b3ab6187262c2Timo Sirainen sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->space_diff = extra_space;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen } else {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->expunged_space = 0;
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen }
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mbox_sync_rewrite(sync_ctx, sync_ctx->need_space_seq, sync_ctx->seq,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->space_diff) < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen update_from_offsets(sync_ctx->ibox, sync_ctx->t, sync_ctx->mails,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->need_space_seq, sync_ctx->seq);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen don't try to access it */
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync_ctx->need_space_seq = 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen buffer_set_used_size(sync_ctx->mails, 0);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return 0;
705f6fbad395e6f014838e797b7dbcaceafd2f1dTimo Sirainen}
8f5b34c22e4c3bfb35ca13c4744867eb5ddbd3d6Timo Sirainen
8f5b34c22e4c3bfb35ca13c4744867eb5ddbd3d6Timo Sirainenstatic int mbox_sync_parse_all(struct mbox_sync_context *sync_ctx,
8f5b34c22e4c3bfb35ca13c4744867eb5ddbd3d6Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen{
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen const struct mail_index_record *rec;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uint32_t uid, messages_count;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen uoff_t offset;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen int ret, expunged;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
2eb0402a28bd0422e0170160808c67d6c7274689Timo Sirainen sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
97180ea9c26c4de0807daaad21e03c80643b09fdTimo Sirainen sync_ctx->input = sync_ctx->ibox->mbox_stream;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->fd = sync_ctx->ibox->mbox_fd;
2eb0402a28bd0422e0170160808c67d6c7274689Timo Sirainen
2eb0402a28bd0422e0170160808c67d6c7274689Timo Sirainen istream_raw_mbox_seek(sync_ctx->input, 0);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen uid = mail_ctx->mail.uid;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen /* get all sync records related to this message */
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (!expunged)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen ret = mbox_sync_handle_header(mail_ctx);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen else {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mail_ctx->mail.uid = 0;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen ret = mbox_sync_handle_expunge(mail_ctx);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (ret < 0) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen /* -1 = error, -2 = need exclusive lock */
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return ret;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_sync_read_index_rec(sync_ctx, uid, &rec) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (!expunged) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_sync_update_index(sync_ctx, &mail_ctx->mail,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen rec) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen istream_raw_mbox_next(sync_ctx->input,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mail_ctx->mail.body_size);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (sync_ctx->need_space_seq != 0) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen i_stream_seek(sync_ctx->input, offset);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen } else if (sync_ctx->expunged_space > 0 && !expunged) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen /* move the body */
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_move(sync_ctx,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mail_ctx->body_offset -
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->expunged_space,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mail_ctx->body_offset,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mail_ctx->mail.body_size) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen i_stream_seek(sync_ctx->input, offset);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen /* rest of the messages in index don't exist -> expunge them */
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen while (sync_ctx->idx_seq < messages_count)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mail_index_expunge(sync_ctx->t, ++sync_ctx->idx_seq);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return 0;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen}
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen{
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen uoff_t offset, extra_space, trailer_size;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen trailer_size = i_stream_get_size(sync_ctx->file_input) -
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->file_input->v_offset;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (sync_ctx->need_space_seq != 0) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen i_assert(sync_ctx->space_diff < 0);
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen extra_space = MBOX_HEADER_EXTRA_SPACE *
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->space_diff -= extra_space;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->expunged_space -= -sync_ctx->space_diff;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mail_ctx->have_eoh && !mail_ctx->updated)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen str_append_c(mail_ctx->header, '\n');
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (sync_ctx->space_diff < 0 &&
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mbox_sync_grow_file(sync_ctx, mail_ctx,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen -sync_ctx->space_diff) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (sync_ctx->seq != sync_ctx->need_space_seq) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen buffer_set_used_size(sync_ctx->mails,
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen (sync_ctx->seq -
7744586e3e0fd60158abfbb03a233d3bd8d6c48bTimo Sirainen sync_ctx->need_space_seq) *
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sizeof(mail_ctx->mail));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sizeof(mail_ctx->mail));
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mbox_sync_rewrite(sync_ctx,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->need_space_seq,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->seq, extra_space) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen update_from_offsets(sync_ctx->ibox, sync_ctx->t,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->mails,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->need_space_seq,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->seq);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (sync_ctx->expunged_space > 0) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen /* copy trailer, then truncate the file */
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen offset = i_stream_get_size(sync_ctx->file_input) -
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->expunged_space - trailer_size;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen if (mbox_move(sync_ctx, offset,
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen offset + sync_ctx->expunged_space,
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen trailer_size) < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (ftruncate(sync_ctx->ibox->mbox_fd,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen offset + trailer_size) < 0) {
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
c5a6a6565be93224fc26522eda855b0990f256e8Timo Sirainen istream_raw_mbox_flush(sync_ctx->input);
f934b271c69c7b3e5e3bca23ff9b3ab6187262c2Timo Sirainen }
97180ea9c26c4de0807daaad21e03c80643b09fdTimo Sirainen return 0;
97180ea9c26c4de0807daaad21e03c80643b09fdTimo Sirainen}
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen{
d176f84ce5ca2073f4dfbafb457b9c74f6bf0d76Timo Sirainen struct stat st;
c5a6a6565be93224fc26522eda855b0990f256e8Timo Sirainen
c5a6a6565be93224fc26522eda855b0990f256e8Timo Sirainen if (fstat(sync_ctx->ibox->mbox_fd, &st) < 0) {
c5a6a6565be93224fc26522eda855b0990f256e8Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
c5a6a6565be93224fc26522eda855b0990f256e8Timo Sirainen return -1;
c5a6a6565be93224fc26522eda855b0990f256e8Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mail_index_update_header(sync_ctx->t,
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen offsetof(struct mail_index_header, uid_validity),
fc464e5b2b2ab4d415a5d5b90ce4475d34620a75Timo Sirainen &sync_ctx->base_uid_validity,
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen sizeof(sync_ctx->base_uid_validity));
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen }
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen if (sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen mail_index_update_header(sync_ctx->t,
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen offsetof(struct mail_index_header, next_uid),
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen }
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen uint32_t sync_stamp = st.st_mtime;
705f6fbad395e6f014838e797b7dbcaceafd2f1dTimo Sirainen
b7835adbfddd8c92b51d6653fb759f963302fa78Timo Sirainen mail_index_update_header(sync_ctx->t,
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen offsetof(struct mail_index_header, sync_stamp),
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen &sync_stamp, sizeof(sync_stamp));
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen }
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen uint64_t sync_size = st.st_size;
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen mail_index_update_header(sync_ctx->t,
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen offsetof(struct mail_index_header, sync_size),
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen &sync_size, sizeof(sync_size));
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen }
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen return 0;
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen}
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen{
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->base_uid_validity = 0;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen sync_ctx->base_uid_last = 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->next_uid = 1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen sync_ctx->prev_msg_uid = sync_ctx->first_uid = 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->seq = sync_ctx->idx_seq = 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen mail_index_transaction_rollback(sync_ctx->t);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen}
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainenstatic int mbox_sync_do(struct mbox_sync_context *sync_ctx, int index_synced)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen{
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen struct index_mailbox *ibox = sync_ctx->ibox;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen struct mbox_sync_mail_context mail_ctx;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen int ret, lock_type;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen lock_type = mail_index_sync_have_more(sync_ctx->index_sync_ctx) ?
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen F_WRLCK : F_RDLCK;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if ((ret = mbox_lock(ibox, lock_type, &sync_ctx->lock_id)) <= 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return ret;
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mbox_file_open_stream(ibox) < 0)
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen return -1;
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen if ((ret = mbox_sync_parse_all(sync_ctx, &mail_ctx)) == -1)
cf0ad1a0bddb0787f3d7b408a96d721a8b2a98a3Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (ret == -2) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen /* we want to modify mbox, get exclusive lock. this requires
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen dropping the read lock first, so we have to parse the whole
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mbox again */
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx->lock_id = 0;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if ((ret = mbox_lock(ibox, F_WRLCK, &sync_ctx->lock_id)) <= 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return ret;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mbox_file_open_stream(ibox) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen /* FIXME: if mbox timestamp hasn't changed and it's older than
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen 2 secs, we could continue from where we left */
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen mbox_sync_restart(sync_ctx);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen if (mbox_sync_parse_all(sync_ctx, &mail_ctx) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen }
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
abb9b8f14e83baca887cf12210bfe480f6cde7a6Timo Sirainen if (sync_ctx->base_uid_last+1 != sync_ctx->next_uid) {
abb9b8f14e83baca887cf12210bfe480f6cde7a6Timo Sirainen // FIXME: rewrite X-IMAPbase header
abb9b8f14e83baca887cf12210bfe480f6cde7a6Timo Sirainen }
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen /* only syncs left should be just appends (and their updates)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen which weren't synced yet for some reason (crash). we'll just
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen ignore them, as we've overwritten them above. */
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen while (mail_index_sync_next(sync_ctx->index_sync_ctx,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen &sync_ctx->sync_rec) > 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen ;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mbox_sync_update_index_header(sync_ctx) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen}
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainenstatic int mbox_sync_has_changed(struct index_mailbox *ibox)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen{
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen const struct mail_index_header *hdr;
9ba7e76c20be775f368254e3059a6189fe789f16Timo Sirainen struct stat st;
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen if (mail_index_get_header(ibox->view, &hdr) < 0) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mail_storage_set_index_error(ibox);
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen return -1;
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen }
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (stat(ibox->path, &st) < 0) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen mbox_set_syscall_error(ibox, "stat()");
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen }
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen return (uint32_t)st.st_mtime != hdr->sync_stamp ||
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen (uint64_t)st.st_size != hdr->sync_size;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen}
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainenint mbox_sync(struct index_mailbox *ibox, int last_commit)
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen{
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen struct mail_index_sync_ctx *index_sync_ctx;
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen struct mail_index_view *sync_view;
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen struct mbox_sync_context sync_ctx;
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen uint32_t seq;
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen uoff_t offset;
fa2a11210f20fb8998ed656f75e163191c8047e6Timo Sirainen int ret, index_synced;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if ((ret = mbox_sync_has_changed(ibox)) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (ret == 0 && !last_commit)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return 0;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen index_synced = ret > 0;
705f6fbad395e6f014838e797b7dbcaceafd2f1dTimo Sirainen
8f5b34c22e4c3bfb35ca13c4744867eb5ddbd3d6Timo Sirainen if (last_commit) {
d1fff80640050631b06bfab904a34b2ad24601e8Timo Sirainen seq = ibox->commit_log_file_seq;
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen offset = ibox->commit_log_file_offset;
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen } else {
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen seq = (uint32_t)-1;
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen offset = (uoff_t)-1;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen }
95d9395d15540b3a96f75c7f9fd73e6d8ad5e897Timo Sirainen
95d9395d15540b3a96f75c7f9fd73e6d8ad5e897Timo Sirainen ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen seq, offset);
1b04762685272a53643ac2179939537a44c7c044Timo Sirainen if (ret <= 0) {
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen if (ret < 0)
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen mail_storage_set_index_error(ibox);
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen return ret;
48ada47cce07fb7195a3437224c7c25f542326b0Timo Sirainen }
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen memset(&sync_ctx, 0, sizeof(sync_ctx));
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync_ctx.ibox = ibox;
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen sync_ctx.from_line = str_new(default_pool, 256);
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen sync_ctx.header = str_new(default_pool, 4096);
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen sync_ctx.index_sync_ctx = index_sync_ctx;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx.sync_view = sync_view;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen sync_ctx.next_uid = 1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen i_assert(ret == 0);
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen
f38485358ffc04c3466b917770575e29deef24c3Timo Sirainen if (mbox_sync_do(&sync_ctx, index_synced) < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen ret = -1;
2fb9ae42f9e36388ec6db24188b9108434043fd0Timo Sirainen
47e9fdee55c2074425cf0316f4f64fbbb790301cTimo Sirainen if (ret < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen mail_index_transaction_rollback(sync_ctx.t);
367e28a16854ee9f7247b2518f36f5e9163fcc10Timo Sirainen else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0)
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen ret = -1;
8d3278a82b964217d95c340ec6f82037cdc59d19Timo Sirainen else {
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen ibox->commit_log_file_seq = 0;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen ibox->commit_log_file_offset = 0;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen }
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen if (mail_index_sync_end(index_sync_ctx) < 0)
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen ret = -1;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen if (sync_ctx.lock_id != 0) {
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen ret = -1;
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen }
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen str_free(sync_ctx.header);
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen str_free(sync_ctx.from_line);
62fc2fe221eccc834ac6b11b94b55335d5027cd1Timo Sirainen buffer_free(sync_ctx.mails);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen buffer_free(sync_ctx.syncs);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return ret;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen}
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainenint mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen{
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen struct index_mailbox *ibox = (struct index_mailbox *)box;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen ibox->sync_last_check = ioloop_time;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen if (mbox_sync(ibox, FALSE) < 0)
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return -1;
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen }
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen return index_storage_sync(box, flags);
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen}
4da8c6cdefabd31262318c32da3c13de1d9ea953Timo Sirainen