mail-index-view-sync.c revision 14c474d9f4591c397ed0b5206af6537c7b52c924
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "lib.h"
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "array.h"
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "buffer.h"
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "mail-index-view-private.h"
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "mail-index-sync-private.h"
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#include "mail-transaction-log.h"
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstruct mail_index_view_sync_ctx {
9522aa5f33cc37fe8ccd0d647cc51dd3ba6a9b55Timo Sirainen struct mail_index_view *view;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen enum mail_index_view_sync_flags flags;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ARRAY_TYPE(seq_range) expunges;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct mail_transaction_header *hdr;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const void *data;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen size_t data_offset;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int sync_map_update:1;
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen unsigned int skipped_expunges:1;
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen unsigned int last_read:1;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen};
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic int
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct seq_range *src, size_t src_size)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* Note that all the sequences are actually still UIDs at this point */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct seq_range *src_end;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen struct seq_range *dest, new_exp;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int first, i, dest_count;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_assert(src_size % sizeof(*src) == 0);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* @UNSAFE */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (dest_count == 0) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_append(expunges, src, src_size / sizeof(*src));
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return 0;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen src_end = CONST_PTR_OFFSET(src, src_size);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; src != src_end; src++) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* src[] must be sorted. */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (src->seq1 > src->seq2 ||
cf9d67e4a9bfee31cf3be05244555d51a3d1b9feTimo Sirainen (src+1 != src_end && src->seq2 >= src[1].seq1))
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return -1;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (; i < dest_count; i++) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (src->seq1 < dest[i].seq1)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen break;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen new_exp = *src;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen first = i;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* we can/must merge with next record */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (new_exp.seq2 < dest[i].seq2)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen new_exp.seq2 = dest[i].seq2;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i++;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* continue previous record */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (dest[first-1].seq2 < new_exp.seq2)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dest[first-1].seq2 = new_exp.seq2;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen } else if (i == first) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_insert(expunges, i, &new_exp, 1);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i++; first++;
cf9d67e4a9bfee31cf3be05244555d51a3d1b9feTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen } else {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* use next record */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dest[first] = new_exp;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen first++;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (i > first) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen array_delete(expunges, first, i - first);
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i = first;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return 0;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}
cf9d67e4a9bfee31cf3be05244555d51a3d1b9feTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic int
cf9d67e4a9bfee31cf3be05244555d51a3d1b9feTimo Sirainenview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen{
cf9d67e4a9bfee31cf3be05244555d51a3d1b9feTimo Sirainen const struct mail_index_header *hdr = &view->index->map->hdr;
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen uint32_t start_seq, end_seq;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen uoff_t start_offset, end_offset;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen bool reset;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen int ret;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen start_seq = view->log_file_expunge_seq;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen start_offset = view->log_file_expunge_offset;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen end_seq = hdr->log_file_seq;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen end_offset = hdr->log_file_head_offset;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (;;) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* the view begins from the first non-synced transaction */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ret = mail_transaction_log_view_set(view->log_view,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen start_seq, start_offset,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen end_seq, end_offset,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen &reset);
cf9d67e4a9bfee31cf3be05244555d51a3d1b9feTimo Sirainen if (ret <= 0) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (ret < 0)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return -1;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* FIXME: use the new index to get needed
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen changes */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_set_error(view->index,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen "Transaction log got desynced for index %s",
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->index->filepath);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->inconsistent = TRUE;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return -1;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen }
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen if (!reset || sync_expunges)
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen break;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* we can't do this. sync only up to reset. */
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen &end_seq, &end_offset);
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen end_seq--; end_offset = (uoff_t)-1;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen if (end_seq < start_seq) {
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen /* we have only this reset log */
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen mail_transaction_log_view_clear(view->log_view);
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen break;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen }
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen }
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen return 0;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen}
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainenstatic int
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainenview_sync_get_expunges(struct mail_index_view *view,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen ARRAY_TYPE(seq_range) *expunges_r)
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen{
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen const struct mail_transaction_header *hdr;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen struct seq_range *src, *src_end, *dest;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen const void *data;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen unsigned int count;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen int ret;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen if (view_sync_set_log_view_range(view, TRUE) < 0)
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen return -1;
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen
0a53eb0283d7ec28c6105f61e118b96fce8ecb95Timo Sirainen /* get a list of expunge transactions. there may be some that we have
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen already synced, but it doesn't matter because they'll get dropped
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen out when converting to sequences */
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen i_array_init(expunges_r, 64);
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen &hdr, &data)) > 0) {
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen continue;
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen /* this is simply a request for expunge */
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen continue;
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen }
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen if (mail_transaction_log_sort_expunges(expunges_r, data,
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainen hdr->size) < 0) {
242abe3ad2423776e9cf05e1304eb8fda4831b23Timo Sirainen mail_transaction_log_view_set_corrupted(view->log_view,
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen "Corrupted expunge record");
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen ret = -1;
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen break;
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen }
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen }
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen if (ret < 0) {
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen array_free(expunges_r);
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen return -1;
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen }
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen /* convert UIDs to sequences */
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen src = dest = array_get_modifiable(expunges_r, &count);
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen src_end = src + count;
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen for (; src != src_end; src++) {
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen ret = mail_index_lookup_uid_range(view, src->seq1,
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen src->seq2,
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen &dest->seq1,
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen &dest->seq2);
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen i_assert(ret == 0);
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen if (dest->seq1 == 0)
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen count--;
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen else
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen dest++;
edd318d5866ac3fbc6e8df28fb24a4dfef93c884Timo Sirainen }
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen array_delete(expunges_r, count, array_count(expunges_r) - count);
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen return 0;
b2024fa4e6ed39f9b5b6bb6c051f6d535fc0e011Timo Sirainen}
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen#ifdef DEBUG
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainenstatic void mail_index_view_check(struct mail_index_view *view)
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen{
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen mail_index_map_check(view->map);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen i_assert(view->hdr.messages_count == view->map->hdr.messages_count);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen i_assert(view->hdr.deleted_messages_count ==
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen view->map->hdr.deleted_messages_count);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen i_assert(view->hdr.recent_messages_count ==
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen view->map->hdr.recent_messages_count);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen i_assert(view->hdr.seen_messages_count ==
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen view->map->hdr.seen_messages_count);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen}
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen#endif
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen#define VIEW_IS_SYNCED_TO_SAME(hdr, tail_seq, tail_offset) \
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen ((hdr)->log_file_seq == (tail_seq) && \
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen (hdr)->log_file_head_offset == (tail_offset))
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainenint mail_index_view_sync_begin(struct mail_index_view *view,
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen enum mail_index_view_sync_flags flags,
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen struct mail_index_view_sync_ctx **ctx_r)
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen{
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen struct mail_index_view_sync_ctx *ctx;
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen struct mail_index_map *map;
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen ARRAY_TYPE(seq_range) expunges = ARRAY_INIT;
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen bool sync_expunges;
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen i_assert(!view->syncing);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen i_assert(view->transactions == 0);
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen if (mail_index_map_lock(view->index->map) < 0)
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen return -1;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen if (sync_expunges) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* get list of all expunges first */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (view_sync_get_expunges(view, &expunges) < 0)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return -1;
d2cadbf5445156fc12988506279d51d0e53b0449Timo Sirainen }
d2cadbf5445156fc12988506279d51d0e53b0449Timo Sirainen
d2cadbf5445156fc12988506279d51d0e53b0449Timo Sirainen if (view_sync_set_log_view_range(view, sync_expunges) < 0) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (array_is_created(&expunges))
d2cadbf5445156fc12988506279d51d0e53b0449Timo Sirainen array_free(&expunges);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return -1;
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen }
f158d9a303bb15a6848ca276c9391c7ca52e452bTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ctx = i_new(struct mail_index_view_sync_ctx, 1);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ctx->view = view;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ctx->flags = flags;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen ctx->expunges = expunges;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_sync_map_init(&ctx->sync_map_ctx, view,
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen MAIL_INDEX_SYNC_HANDLER_VIEW);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (sync_expunges) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->sync_new_map = view->index->map;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen view->sync_new_map->refcount++;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen /* keep the old mapping without expunges until we're
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen fully synced */
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen } else {
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen /* We need a private copy of the map if we don't want to
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen sync expunges.
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen If view's map is the head map, it means that it contains
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen already all the latest changes and there's no need for us
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen to apply any changes to it. This can only happen if there
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen hadn't been any expunges. */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (view->map != view->index->map) {
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen /* Using non-head mapping. We have to apply
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen transactions to it to get latest changes into it. */
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen ctx->sync_map_update = TRUE;
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen }
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen /* Unless map was synced at the exact same position as
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen view, the message flags can't be reliably used to
5e9bb72de1209cd39fdf3e95bdb26e047cc5594eTimo Sirainen update flag counters. note that map->hdr may contain
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen old information if another process updated the
caa1fa99c79c568ce2e42477bc169e7024fb220bTimo Sirainen index file since. */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (view->map->mmap_base != NULL) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen // FIXME: locking should do this..?..
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct mail_index_header *hdr;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen hdr = view->map->mmap_base;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->map->hdr = *hdr;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (view->map->refcount > 1) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen map = mail_index_map_clone(view->map);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_unmap(&view->map);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->map = map;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen } else {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen map = view->map;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->hdr = map->hdr;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen i_assert(map->records_count == map->hdr.messages_count);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#ifdef DEBUG
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_view_check(view);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen#endif
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen /* Syncing the view invalidates all previous looked up records.
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen Unreference the mappings this view keeps because of them. */
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen mail_index_view_unref_maps(view);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen view->syncing = TRUE;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen *ctx_r = ctx;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return 0;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenstatic bool
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainenview_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
bcbca4b66800f0fbfe2643a86d5a6b63d752454dTimo Sirainen{
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen const struct mail_index_view_log_sync_area *syncs;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen unsigned int i, count;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (!array_is_created(&view->syncs_hidden))
9522aa5f33cc37fe8ccd0d647cc51dd3ba6a9b55Timo Sirainen return FALSE;
9522aa5f33cc37fe8ccd0d647cc51dd3ba6a9b55Timo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen syncs = array_get(&view->syncs_hidden, &count);
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen for (i = 0; i < count; i++) {
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen if (syncs[i].log_file_offset <= offset &&
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen offset - syncs[i].log_file_offset < syncs[i].length &&
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen syncs[i].log_file_seq == seq)
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return TRUE;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen }
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen return FALSE;
2670cd577aa57eb9f915a4f4220ae48c9b4fc5fbTimo Sirainen}
static bool
mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
const struct mail_transaction_header *hdr)
{
struct mail_index_view *view = ctx->view;
uint32_t seq;
uoff_t offset, next_offset;
mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
next_offset = offset + sizeof(*hdr) + hdr->size;
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
(hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
i_assert(!LOG_IS_BEFORE(seq, offset,
view->log_file_expunge_seq,
view->log_file_expunge_offset));
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = seq;
view->log_file_expunge_offset = offset;
ctx->skipped_expunges = TRUE;
}
return FALSE;
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
view->log_file_expunge_offset)) {
/* already synced */
return FALSE;
}
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
view->log_file_head_offset)) {
/* already synced */
return FALSE;
}
view->log_file_head_seq = seq;
view->log_file_head_offset = next_offset;
return TRUE;
}
static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
struct mail_transaction_log_view *log_view = ctx->view->log_view;
struct mail_index_view *view = ctx->view;
const struct mail_transaction_header *hdr;
uint32_t seq;
uoff_t offset;
int ret;
bool synced_to_map;
for (;;) {
/* Get the next transaction from log. */
ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
&ctx->data);
if (ret <= 0) {
if (ret < 0)
return -1;
ctx->hdr = NULL;
ctx->last_read = TRUE;
return 0;
}
hdr = ctx->hdr;
if (!mail_index_view_sync_want(ctx, hdr)) {
/* This is a visible record that we don't want to
sync. */
continue;
}
mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
/* If we started from a map that we didn't create ourself,
some of the transactions may already be synced. at the end
of this view sync we'll update file_seq=0 so that this check
always becomes FALSE for subsequent syncs. */
synced_to_map = view->map->hdr.log_file_seq != 0 &&
!LOG_IS_BEFORE(seq, offset,
view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset);
/* Apply transaction to view's mapping if needed (meaning we
didn't just re-map the view to head mapping). */
if (ctx->sync_map_update && !synced_to_map) {
i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0 ||
(hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0);
if (mail_index_sync_record(&ctx->sync_map_ctx,
hdr, ctx->data) < 0)
return -1;
}
/* skip changes committed by hidden transactions (eg. in IMAP
store +flags.silent command) */
if (view_sync_is_hidden(view, seq, offset))
continue;
break;
}
return 1;
}
#define FLAG_UPDATE_IS_INTERNAL(u) \
((((u)->add_flags | (u)->remove_flags) & \
~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)
static bool
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *rec)
{
const struct mail_transaction_header *hdr = ctx->hdr;
const void *data = ctx->data;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
/* data contains the appended records, but we don't care */
rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
rec->uid1 = rec->uid2 = 0;
ctx->data_offset += hdr->size;
break;
}
case MAIL_TRANSACTION_EXPUNGE: {
const struct mail_transaction_expunge *exp =
CONST_PTR_OFFSET(data, ctx->data_offset);
if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
/* this is simply a request for expunge */
ctx->data_offset = ctx->hdr->size;
return 0;
}
/* data contains mail_transaction_expunge[] */
rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
rec->uid1 = exp->uid1;
rec->uid2 = exp->uid2;
ctx->data_offset += sizeof(*exp);
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_flag_update[] */
for (;;) {
ctx->data_offset += sizeof(*update);
if (!FLAG_UPDATE_IS_INTERNAL(update))
break;
/* skip internal flag changes */
if (ctx->data_offset == ctx->hdr->size)
return 0;
update = CONST_PTR_OFFSET(data, ctx->data_offset);
}
rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
rec->uid1 = update->uid1;
rec->uid2 = update->uid2;
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *update = data;
const uint32_t *uids;
/* data contains mail_transaction_keyword_update header,
the keyword name and an array of { uint32_t uid1, uid2; } */
if (ctx->data_offset == 0) {
/* skip over the header and name */
ctx->data_offset = sizeof(*update) + update->name_size;
if ((ctx->data_offset % 4) != 0)
ctx->data_offset += 4 - (ctx->data_offset % 4);
}
uids = CONST_PTR_OFFSET(data, ctx->data_offset);
rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
rec->uid1 = uids[0];
rec->uid2 = uids[1];
ctx->data_offset += sizeof(uint32_t) * 2;
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *reset =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_keyword_reset[] */
rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
rec->uid1 = reset->uid1;
rec->uid2 = reset->uid2;
ctx->data_offset += sizeof(*reset);
break;
}
default:
ctx->hdr = NULL;
return FALSE;
}
return TRUE;
}
int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
int ret;
do {
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ret = mail_index_view_sync_get_next_transaction(ctx);
if (ret <= 0)
return ret;
ctx->data_offset = 0;
}
} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
return 1;
}
void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
const ARRAY_TYPE(seq_range) **expunges_r)
{
*expunges_r = &ctx->expunges;
}
static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
{
const struct mail_index_view_log_sync_area *syncs;
unsigned int i, count;
if (!array_is_created(&view->syncs_hidden))
return;
/* Clean up to view's tail */
syncs = array_get(&view->syncs_hidden, &count);
for (i = 0; i < count; i++) {
if ((syncs[i].log_file_offset +
syncs[i].length > view->log_file_expunge_offset &&
syncs[i].log_file_seq == view->log_file_expunge_seq) ||
syncs[i].log_file_seq > view->log_file_expunge_seq)
break;
}
if (i > 0)
array_delete(&view->syncs_hidden, 0, i);
}
void mail_index_view_sync_end(struct mail_index_view_sync_ctx **_ctx)
{
struct mail_index_view_sync_ctx *ctx = *_ctx;
struct mail_index_view *view = ctx->view;
i_assert(view->syncing);
*_ctx = NULL;
if (!ctx->last_read) {
/* we didn't sync everything */
view->inconsistent = TRUE;
}
if (view->sync_new_map != NULL) {
mail_index_unmap(&view->map);
view->map = view->sync_new_map;
view->sync_new_map = NULL;
}
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = view->log_file_head_seq;
view->log_file_expunge_offset = view->log_file_head_offset;
}
if (ctx->sync_map_update) {
/* log offsets have no meaning in views. make sure they're not
tried to be used wrong by setting them to zero. */
view->map->hdr.log_file_seq = 0;
view->map->hdr.log_file_head_offset = 0;
view->map->hdr.log_file_tail_offset = 0;
}
view->hdr = view->map->hdr;
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
mail_index_view_sync_clean_log_syncs(ctx->view);
#ifdef DEBUG
mail_index_view_check(view);
#endif
/* set log view to empty range so unneeded memory gets freed */
mail_transaction_log_view_clear(view->log_view);
if (array_is_created(&ctx->expunges))
array_free(&ctx->expunges);
view->syncing = FALSE;
i_free(ctx);
}
void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset,
unsigned int length)
{
struct mail_index_view_log_sync_area *area;
if (!array_is_created(&view->syncs_hidden))
i_array_init(&view->syncs_hidden, 32);
area = array_append_space(&view->syncs_hidden);
area->log_file_seq = log_file_seq;
area->log_file_offset = log_file_offset;
area->length = length;
}