mail-index-view-sync.c revision 4e1899e9bdafd0da1692240d21314d9c07d9946e
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
08d6658a4e2ec8104cd1307f6baa75fdb07a24f8Mark Washenberger#include "lib.h"
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen#include "array.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "buffer.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-index-view-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-index-sync-private.h"
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen#include "mail-index-modseq.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-transaction-log.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include <stdlib.h>
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstruct mail_index_view_sync_ctx {
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen struct mail_index_view *view;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen enum mail_index_view_sync_flags flags;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen /* After syncing view, map is replaced with sync_new_map. */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index_map *sync_new_map;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ARRAY_TYPE(seq_range) expunges;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int finish_min_msg_count;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const struct mail_transaction_header *hdr;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const void *data;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* temporary variables while handling lost transaction logs: */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen buffer_t *lost_kw_buf;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen uint32_t lost_new_ext_idx;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* result of lost transaction logs: */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ARRAY_TYPE(seq_range) lost_flags;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen unsigned int lost_flag_idx;
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen size_t data_offset;
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen unsigned int failed:1;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen unsigned int sync_map_update:1;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen unsigned int skipped_expunges:1;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen unsigned int last_read:1;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen unsigned int log_was_lost:1;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen unsigned int hidden:1;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen};
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainenstatic int
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen const struct seq_range *src, size_t src_size)
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen{
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen /* Note that all the sequences are actually still UIDs at this point */
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen const struct seq_range *src_end;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen struct seq_range *dest, new_exp;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen unsigned int first, i, dest_count;
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen i_assert(src_size % sizeof(*src) == 0);
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* @UNSAFE */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen dest = array_get_modifiable(expunges, &dest_count);
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen if (dest_count == 0) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen array_append(expunges, src, src_size / sizeof(*src));
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen return 0;
0f39a57760d93cddbce3ca43096d78e0fe2f42fdTimo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen src_end = CONST_PTR_OFFSET(src, src_size);
0f39a57760d93cddbce3ca43096d78e0fe2f42fdTimo Sirainen for (i = 0; src != src_end; src++) {
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen /* src[] must be sorted. */
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen if (src->seq1 > src->seq2 ||
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen (src+1 != src_end && src->seq2 >= src[1].seq1))
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen return -1;
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen for (; i < dest_count; i++) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen if (src->seq1 < dest[i].seq1)
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen break;
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen }
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen new_exp = *src;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen first = i;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we can/must merge with next record */
0f39a57760d93cddbce3ca43096d78e0fe2f42fdTimo Sirainen if (new_exp.seq2 < dest[i].seq2)
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen new_exp.seq2 = dest[i].seq2;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i++;
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen }
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* continue previous record */
6fabfb7bbfd88d0c1de66981e52850f26067623bTimo Sirainen if (dest[first-1].seq2 < new_exp.seq2)
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen dest[first-1].seq2 = new_exp.seq2;
a30b52af112bc98b74b8624e9a5d20cb754b2ab7Timo Sirainen } else if (i == first) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_insert(expunges, i, &new_exp, 1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i++; first++;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen dest = array_get_modifiable(expunges, &dest_count);
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen } else {
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen /* use next record */
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen dest[first] = new_exp;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen first++;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen }
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen if (i > first) {
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen array_delete(expunges, first, i - first);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i = first;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen }
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen }
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic int
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainenview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen bool *reset_r)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen const struct mail_index_header *hdr = &view->index->map->hdr;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t start_seq, end_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uoff_t start_offset, end_offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen int ret;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen start_seq = view->log_file_expunge_seq;
a7bee3930831a9261fa6180d02af29c484d862e9Timo Sirainen start_offset = view->log_file_expunge_offset;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen end_seq = hdr->log_file_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen end_offset = hdr->log_file_head_offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen for (;;) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* the view begins from the first non-synced transaction */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen ret = mail_transaction_log_view_set(view->log_view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen start_seq, start_offset,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen end_seq, end_offset,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen reset_r);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (ret <= 0)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return ret;
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (!*reset_r || sync_expunges)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen break;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* log was reset, but we don't want to sync expunges.
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen we can't do this, so sync only up to the reset. */
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen &end_seq, &end_offset);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen end_seq--; end_offset = (uoff_t)-1;
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen if (end_seq < start_seq) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* we have only this reset log */
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen mail_transaction_log_view_clear(view->log_view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen view->log_file_expunge_seq);
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen break;
7af2a19a89c1c9da8848c570190d36570afd09e6Timo Sirainen }
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen }
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen return 1;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen}
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainenstatic unsigned int
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenview_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen struct mail_index_view *view = ctx->view;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct seq_range *src, *src_end, *dest;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen unsigned int count, expunge_count = 0;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen uint32_t prev_seq = 0;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* convert UIDs to sequences */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen src = dest = array_get_modifiable(&ctx->expunges, &count);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen src_end = src + count;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen for (; src != src_end; src++) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen &dest->seq1, &dest->seq2))
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen count--;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen else {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen i_assert(dest->seq1 > prev_seq);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen prev_seq = dest->seq2;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen expunge_count += dest->seq2 - dest->seq1 + 1;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen dest++;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen array_delete(&ctx->expunges, count,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen array_count(&ctx->expunges) - count);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return expunge_count;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen}
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenstatic int
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenview_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen unsigned int *expunge_count_r)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen struct mail_index_view *view = ctx->view;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const struct mail_transaction_header *hdr;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const void *data;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen int ret;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* get a list of expunge transactions. there may be some that we have
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen already synced, but it doesn't matter because they'll get dropped
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen out when converting to sequences */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen mail_transaction_log_view_mark(view->log_view);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen &hdr, &data)) > 0) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen continue;
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen /* this is simply a request for expunge */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen continue;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (mail_transaction_log_sort_expunges(&ctx->expunges, data,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen hdr->size) < 0) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen mail_transaction_log_view_set_corrupted(view->log_view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen "Corrupted expunge record");
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return -1;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen mail_transaction_log_view_rewind(view->log_view);
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen *expunge_count_r = view_sync_expunges2seqs(ctx);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return 0;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen}
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainenstatic bool have_existing_expunges(struct mail_index_view *view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const struct seq_range *range, size_t size)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const struct seq_range *range_end;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen uint32_t seq1, seq2;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen range_end = CONST_PTR_OFFSET(range, size);
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen for (; range < range_end; range++) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen &seq1, &seq2))
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen return TRUE;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen }
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen return FALSE;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen}
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainenstatic bool view_sync_have_expunges(struct mail_index_view *view)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen{
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const struct mail_transaction_header *hdr;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen const void *data;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen bool have_expunges = FALSE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen int ret;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen mail_transaction_log_view_mark(view->log_view);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen &hdr, &data)) > 0) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
e9371f899a3d4207a0ffd3923ea5ec7250cf5e75Timo Sirainen continue;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* this is simply a request for expunge */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen continue;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen /* we have an expunge. see if it still exists. */
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (have_existing_expunges(view, data, hdr->size)) {
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen have_expunges = TRUE;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen break;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen }
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen mail_transaction_log_view_rewind(view->log_view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen /* handle failures as having expunges (which is safer).
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen we'll probably fail later. */
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen return ret < 0 || have_expunges;
e665999b757e60bfb98e5a84a78b05f061453140Timo Sirainen}
c5383a0ed56a188a7d5efaaa4c6f8243af432d65Timo Sirainen
74674a53a72dab535c61f455b2246ef2797844eaTimo Sirainenstatic int uint_cmp(const void *p1, const void *p2)
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen{
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen const unsigned int *u1 = p1, *u2 = p2;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (*u1 < *u2)
f3d506e525a720f214020ca0f989a1966b30edaeTimo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (*u1 > *u2)
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen return 1;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen return 0;
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen}
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstatic bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen{
f8a86fdfb0048f9c87bf223373b35416ceb5856bTimo Sirainen unsigned int *old_idx, *new_idx;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen unsigned int old_count, new_count;
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (old_count != new_count)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return FALSE;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen return memcmp(old_idx, new_idx, old_count * sizeof(old_idx)) == 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainenstatic int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen uint32_t uid)
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_transaction_header thdr;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_transaction_keyword_update kw_up;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const unsigned int *kw_idx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const char *const *kw_names;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen unsigned int i, count;
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen kw_idx = array_get(&ctx->lost_new_kw, &count);
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen if (count == 0)
43d3ea2780b5f8557ede7b4c039e8f56cb8d357dTimo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen kw_names = array_idx(&ctx->view->index->keywords, 0);
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen memset(&thdr, 0, sizeof(thdr));
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen memset(&kw_up, 0, sizeof(kw_up));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen kw_up.modify_type = MODIFY_ADD;
/* add new flags one by one */
for (i = 0; i < count; i++) {
kw_up.name_size = strlen(kw_names[kw_idx[i]]);
buffer_set_used_size(ctx->lost_kw_buf, 0);
buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
kw_up.name_size);
if (ctx->lost_kw_buf->used % 4 != 0) {
buffer_append_zero(ctx->lost_kw_buf,
4 - ctx->lost_kw_buf->used % 4);
}
buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
thdr.size = ctx->lost_kw_buf->used;
if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
ctx->lost_kw_buf->data) < 0)
return -1;
}
return 0;
}
static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
uint32_t old_seq, uint32_t new_seq)
{
struct mail_index_map *old_map = ctx->view->map;
struct mail_index_map *new_map = ctx->view->index->map;
const struct mail_index_record *old_rec, *new_rec;
struct mail_transaction_header thdr;
const struct mail_index_ext *ext;
const uint64_t *modseqp;
uint64_t new_modseq;
bool changed = FALSE;
old_rec = MAIL_INDEX_MAP_IDX(old_map, old_seq - 1);
new_rec = MAIL_INDEX_MAP_IDX(new_map, new_seq - 1);
memset(&thdr, 0, sizeof(thdr));
if (old_rec->flags != new_rec->flags) {
struct mail_transaction_flag_update flag_update;
/* check this before syncing the record, since it updates
old_rec. */
if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
(new_rec->flags & MAIL_INDEX_FLAGS_MASK))
changed = TRUE;
thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
MAIL_TRANSACTION_EXTERNAL;
thdr.size = sizeof(flag_update);
memset(&flag_update, 0, sizeof(flag_update));
flag_update.uid1 = flag_update.uid2 = new_rec->uid;
flag_update.add_flags = new_rec->flags;
flag_update.remove_flags = ~new_rec->flags & 0xff;
if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
&flag_update) < 0)
return -1;
}
mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw);
mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw);
if (!view_sync_lost_keywords_equal(ctx)) {
struct mail_transaction_keyword_reset kw_reset;
thdr.type = MAIL_TRANSACTION_KEYWORD_RESET |
MAIL_TRANSACTION_EXTERNAL;
thdr.size = sizeof(kw_reset);
/* remove all old flags by resetting them */
memset(&kw_reset, 0, sizeof(kw_reset));
kw_reset.uid1 = kw_reset.uid2 = new_rec->uid;
if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
&kw_reset) < 0)
return -1;
view_sync_update_keywords(ctx, new_rec->uid);
changed = TRUE;
}
if (changed) {
/* flags or keywords changed */
} else if (ctx->view->highest_modseq != 0 &&
ctx->lost_new_ext_idx != (uint32_t)-1) {
/* if modseq has changed include this message in changed flags
list, even if we didn't see any changes above. */
ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
new_modseq = *modseqp;
if (new_modseq > ctx->view->highest_modseq)
changed = TRUE;
}
/* without modseqs lost_flags isn't updated perfectly correctly, because
by the time we're comparing old flags it may have changed from what
we last sent to the client (because the map is shared). This could
be avoided by always keeping a private copy of the map in the view,
but that's a waste of memory for as rare of a problem as this. */
if (changed)
seq_range_array_add(&ctx->lost_flags, 0, new_rec->uid);
return 0;
}
static int
view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
unsigned int *expunge_count_r)
{
struct mail_index_view *view = ctx->view;
struct mail_index_map *old_map = view->map;
struct mail_index_map *new_map = view->index->map;
const unsigned int old_count = old_map->hdr.messages_count;
const unsigned int new_count = new_map->hdr.messages_count;
const struct mail_index_record *old_rec, *new_rec;
struct mail_transaction_header thdr;
unsigned int i, j;
/* we don't update the map in the same order as it's typically done.
map->rec_map may already have some messages appended that we don't
want. get an atomic map to make sure these get removed. */
(void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
&ctx->lost_new_ext_idx))
ctx->lost_new_ext_idx = (uint32_t)-1;
i_array_init(&ctx->lost_flags, 64);
t_array_init(&ctx->lost_old_kw, 32);
t_array_init(&ctx->lost_new_kw, 32);
ctx->lost_kw_buf = buffer_create_dynamic(pool_datastack_create(), 128);
/* handle expunges and sync flags */
i = j = 0;
while (i < old_count && j < new_count) {
old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
if (old_rec->uid == new_rec->uid) {
/* message found - check if flags have changed */
if (view_sync_apply_lost_changes(ctx, i + 1, j + 1) < 0)
return -1;
i++; j++;
} else if (old_rec->uid < new_rec->uid) {
/* message expunged */
seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
i++;
} else {
/* new message appeared out of nowhere */
mail_index_set_error(view->index,
"%s view is inconsistent: "
"uid=%u inserted in the middle of mailbox",
view->index->filepath, new_rec->uid);
return -1;
}
}
/* if there are old messages left, they're all expunged */
for (; i < old_count; i++) {
old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
}
/* if there are new messages left, they're all new messages */
thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL;
thdr.size = sizeof(*new_rec);
for (; j < new_count; j++) {
new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
if (mail_index_sync_record(&ctx->sync_map_ctx,
&thdr, new_rec) < 0)
return -1;
mail_index_map_lookup_keywords(new_map, j + 1,
&ctx->lost_new_kw);
view_sync_update_keywords(ctx, new_rec->uid);
}
*expunge_count_r = view_sync_expunges2seqs(ctx);
/* we have no idea how far we've synced - make sure these aren't used */
old_map->hdr.log_file_seq = 0;
old_map->hdr.log_file_head_offset = 0;
old_map->hdr.log_file_tail_offset = 0;
if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
array_clear(&ctx->expunges);
ctx->skipped_expunges = *expunge_count_r > 0;
} else {
view->log_file_head_seq = new_map->hdr.log_file_seq;
view->log_file_head_offset = new_map->hdr.log_file_head_offset;
}
return 0;
}
static int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx)
{
struct mail_index_view *view = ctx->view;
uint32_t seq;
uoff_t offset;
bool reset;
/* replace the view's map */
view->index->map->refcount++;
mail_index_unmap(&view->map);
view->map = view->index->map;
/* update log positions */
view->log_file_head_seq = seq = view->map->hdr.log_file_seq;
view->log_file_head_offset = offset =
view->map->hdr.log_file_head_offset;
if (mail_transaction_log_view_set(view->log_view, seq, offset,
seq, offset, &reset) < 0)
return -1;
view->inconsistent = FALSE;
return 0;
}
struct mail_index_view_sync_ctx *
mail_index_view_sync_begin(struct mail_index_view *view,
enum mail_index_view_sync_flags flags)
{
struct mail_index_view_sync_ctx *ctx;
struct mail_index_map *map;
unsigned int expunge_count = 0;
bool reset, sync_expunges, have_expunges;
int ret;
i_assert(!view->syncing);
i_assert(view->transactions == 0);
view->syncing = TRUE;
/* Syncing the view invalidates all previous looked up records.
Unreference the mappings this view keeps because of them. */
mail_index_view_unref_maps(view);
ctx = i_new(struct mail_index_view_sync_ctx, 1);
ctx->view = view;
ctx->flags = flags;
sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
if (sync_expunges)
i_array_init(&ctx->expunges, 64);
if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
/* just get this view synced - don't return anything */
i_assert(sync_expunges);
if (mail_index_view_sync_init_fix(ctx) < 0)
ctx->failed = TRUE;
return ctx;
}
if (mail_index_view_is_inconsistent(view)) {
mail_index_set_error(view->index, "%s view is inconsistent",
view->index->filepath);
ctx->failed = TRUE;
return ctx;
}
ret = view_sync_set_log_view_range(view, sync_expunges, &reset);
if (ret < 0) {
ctx->failed = TRUE;
return ctx;
}
if (ret == 0) {
ctx->log_was_lost = TRUE;
if (!sync_expunges)
i_array_init(&ctx->expunges, 64);
mail_index_sync_map_init(&ctx->sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_VIEW);
ret = view_sync_get_log_lost_changes(ctx, &expunge_count);
mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
if (ret < 0) {
mail_index_set_error(view->index,
"%s view syncing failed to apply changes",
view->index->filepath);
view->inconsistent = TRUE;
ctx->failed = TRUE;
return ctx;
}
have_expunges = expunge_count > 0;
} else if (sync_expunges) {
/* get list of all expunges first */
if (view_sync_get_expunges(ctx, &expunge_count) < 0) {
ctx->failed = TRUE;
return ctx;
}
have_expunges = expunge_count > 0;
} else {
have_expunges = view_sync_have_expunges(view);
}
ctx->finish_min_msg_count = reset ? 0 :
view->map->hdr.messages_count - expunge_count;
if (reset && view->map->hdr.messages_count > 0) {
view->inconsistent = TRUE;
mail_index_set_error(view->index,
"%s reset, view is now inconsistent",
view->index->filepath);
}
if (!have_expunges) {
/* no expunges, we can just replace the map */
if (view->index->map->hdr.messages_count <
ctx->finish_min_msg_count) {
mail_index_set_error(view->index,
"Index %s lost messages without expunging "
"(%u -> %u)", view->index->filepath,
view->map->hdr.messages_count,
view->index->map->hdr.messages_count);
ctx->finish_min_msg_count = 0;
view->inconsistent = TRUE;
}
view->index->map->refcount++;
mail_index_unmap(&view->map);
view->map = view->index->map;
} else {
/* expunges seen. create a private map which we update.
if we're syncing expunges the map will finally be replaced
with the head map to remove the expunged messages. */
ctx->sync_map_update = TRUE;
if (view->map->refcount > 1) {
map = mail_index_map_clone(view->map);
mail_index_unmap(&view->map);
view->map = map;
} else {
map = view->map;
}
if (sync_expunges) {
ctx->sync_new_map = view->index->map;
ctx->sync_new_map->refcount++;
}
}
mail_index_sync_map_init(&ctx->sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_VIEW);
#ifdef DEBUG
mail_index_map_check(view->map);
#endif
return ctx;
}
static bool
view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
{
const struct mail_index_view_log_sync_area *syncs;
unsigned int i, count;
if (!array_is_created(&view->syncs_hidden))
return FALSE;
syncs = array_get(&view->syncs_hidden, &count);
for (i = 0; i < count; i++) {
if (syncs[i].log_file_offset <= offset &&
offset - syncs[i].log_file_offset < syncs[i].length &&
syncs[i].log_file_seq == seq)
return TRUE;
}
return FALSE;
}
static bool
mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
const struct mail_transaction_header *hdr)
{
struct mail_index_view *view = ctx->view;
uint32_t seq;
uoff_t offset, next_offset;
mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
next_offset = offset + sizeof(*hdr) + hdr->size;
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
(hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
i_assert(!LOG_IS_BEFORE(seq, offset,
view->log_file_expunge_seq,
view->log_file_expunge_offset));
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = seq;
view->log_file_expunge_offset = offset;
ctx->skipped_expunges = TRUE;
}
return FALSE;
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
view->log_file_expunge_offset)) {
/* already synced */
return FALSE;
}
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
view->log_file_head_offset)) {
/* already synced */
return FALSE;
}
view->log_file_head_seq = seq;
view->log_file_head_offset = next_offset;
return TRUE;
}
static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
struct mail_transaction_log_view *log_view = ctx->view->log_view;
struct mail_index_view *view = ctx->view;
const struct mail_transaction_header *hdr;
uint32_t seq;
uoff_t offset;
int ret;
bool synced_to_map;
do {
/* Get the next transaction from log. */
ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
&ctx->data);
if (ret <= 0) {
if (ret < 0)
return -1;
ctx->hdr = NULL;
ctx->last_read = TRUE;
return 0;
}
hdr = ctx->hdr;
/* skip records we've already synced */
} while (!mail_index_view_sync_want(ctx, hdr));
mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
/* If we started from a map that we didn't create ourself,
some of the transactions may already be synced. at the end
of this view sync we'll update file_seq=0 so that this check
always becomes FALSE for subsequent syncs. */
synced_to_map = view->map->hdr.log_file_seq != 0 &&
LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset);
/* Apply transaction to view's mapping if needed (meaning we
didn't just re-map the view to head mapping). */
if (ctx->sync_map_update && !synced_to_map) {
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0) T_BEGIN {
ret = mail_index_sync_record(&ctx->sync_map_ctx,
hdr, ctx->data);
} T_END;
if (ret < 0)
return -1;
}
ctx->hidden = view_sync_is_hidden(view, seq, offset);
return 1;
}
static bool
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *rec)
{
const struct mail_transaction_header *hdr = ctx->hdr;
const void *data = ctx->data;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_flag_update[] */
for (;;) {
ctx->data_offset += sizeof(*update);
if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update))
break;
/* skip internal flag changes */
if (ctx->data_offset == ctx->hdr->size)
return 0;
update = CONST_PTR_OFFSET(data, ctx->data_offset);
}
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = update->uid1;
rec->uid2 = update->uid2;
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *update = data;
const uint32_t *uids;
/* data contains mail_transaction_keyword_update header,
the keyword name and an array of { uint32_t uid1, uid2; } */
if (ctx->data_offset == 0) {
/* skip over the header and name */
ctx->data_offset = sizeof(*update) + update->name_size;
if ((ctx->data_offset % 4) != 0)
ctx->data_offset += 4 - (ctx->data_offset % 4);
}
uids = CONST_PTR_OFFSET(data, ctx->data_offset);
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = uids[0];
rec->uid2 = uids[1];
ctx->data_offset += sizeof(uint32_t) * 2;
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *reset =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_keyword_reset[] */
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = reset->uid1;
rec->uid2 = reset->uid2;
ctx->data_offset += sizeof(*reset);
break;
}
default:
ctx->hdr = NULL;
return FALSE;
}
rec->hidden = ctx->hidden;
return TRUE;
}
static bool
mail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
const struct seq_range *range;
unsigned int count;
range = array_get(&ctx->lost_flags, &count);
if (ctx->lost_flag_idx == count) {
ctx->last_read = TRUE;
return FALSE;
}
sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
sync_rec->uid1 = range[ctx->lost_flag_idx].seq1;
sync_rec->uid2 = range[ctx->lost_flag_idx].seq2;
ctx->lost_flag_idx++;
return TRUE;
}
bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
int ret;
if (ctx->log_was_lost)
return mail_index_view_sync_next_lost(ctx, sync_rec);
do {
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ret = mail_index_view_sync_get_next_transaction(ctx);
if (ret <= 0) {
if (ret < 0)
ctx->failed = TRUE;
return FALSE;
}
ctx->data_offset = 0;
}
} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
return TRUE;
}
void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
const ARRAY_TYPE(seq_range) **expunges_r)
{
*expunges_r = &ctx->expunges;
}
static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
{
const struct mail_index_view_log_sync_area *syncs;
unsigned int i, count;
if (!array_is_created(&view->syncs_hidden))
return;
/* Clean up to view's tail */
syncs = array_get(&view->syncs_hidden, &count);
for (i = 0; i < count; i++) {
if ((syncs[i].log_file_offset +
syncs[i].length > view->log_file_expunge_offset &&
syncs[i].log_file_seq == view->log_file_expunge_seq) ||
syncs[i].log_file_seq > view->log_file_expunge_seq)
break;
}
if (i > 0)
array_delete(&view->syncs_hidden, 0, i);
}
int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
bool *delayed_expunges_r)
{
struct mail_index_view_sync_ctx *ctx = *_ctx;
struct mail_index_view *view = ctx->view;
int ret = ctx->failed ? -1 : 0;
i_assert(view->syncing);
*_ctx = NULL;
*delayed_expunges_r = ctx->skipped_expunges;
if ((!ctx->last_read || view->inconsistent) &&
(ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
/* we didn't sync everything */
view->inconsistent = TRUE;
ret = -1;
}
if (ctx->sync_map_ctx.modseq_ctx != NULL)
mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
if (ctx->sync_new_map != NULL) {
mail_index_unmap(&view->map);
view->map = ctx->sync_new_map;
} else if (ctx->sync_map_update) {
/* log offsets have no meaning in views. make sure they're not
tried to be used wrong by setting them to zero. */
view->map->hdr.log_file_seq = 0;
view->map->hdr.log_file_head_offset = 0;
view->map->hdr.log_file_tail_offset = 0;
}
i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = view->log_file_head_seq;
view->log_file_expunge_offset = view->log_file_head_offset;
}
if (ctx->sync_map_ctx.view != NULL)
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
mail_index_view_sync_clean_log_syncs(ctx->view);
#ifdef DEBUG
mail_index_map_check(view->map);
#endif
/* set log view to empty range so unneeded memory gets freed */
mail_transaction_log_view_clear(view->log_view,
view->log_file_expunge_seq);
if (array_is_created(&ctx->expunges))
array_free(&ctx->expunges);
if (array_is_created(&ctx->lost_flags))
array_free(&ctx->lost_flags);
view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
view->syncing = FALSE;
i_free(ctx);
return ret;
}
void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset,
unsigned int length)
{
struct mail_index_view_log_sync_area *area;
if (!array_is_created(&view->syncs_hidden))
i_array_init(&view->syncs_hidden, 32);
area = array_append_space(&view->syncs_hidden);
area->log_file_seq = log_file_seq;
area->log_file_offset = log_file_offset;
area->length = length;
}