mail-index-view-sync.c revision b6612c334604eeb27e1ca2bd804ac66dcbc2eaad
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen/* Copyright (c) 2003-2008 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "array.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-view-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-sync-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-modseq.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-transaction-log.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include <stdlib.h>
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainenstruct mail_index_view_sync_ctx {
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen struct mail_index_view *view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen enum mail_index_view_sync_flags flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_sync_map_ctx sync_map_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* After syncing view, map is replaced with sync_new_map. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_map *sync_new_map;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen ARRAY_TYPE(seq_range) expunges;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen unsigned int finish_min_msg_count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_transaction_header *hdr;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const void *data;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen /* temporary variables while handling lost transaction logs: */
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen buffer_t *lost_kw_buf;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen uint32_t lost_new_ext_idx;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen /* result of lost transaction logs: */
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen ARRAY_TYPE(seq_range) lost_flags;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen unsigned int lost_flag_idx;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen size_t data_offset;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen unsigned int failed:1;
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen unsigned int sync_map_update:1;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen unsigned int skipped_expunges:1;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen unsigned int last_read:1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen unsigned int log_was_lost:1;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen unsigned int hidden:1;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen};
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainenstatic int
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainenmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct seq_range *src, size_t src_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* Note that all the sequences are actually still UIDs at this point */
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen const struct seq_range *src_end;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen struct seq_range *dest, new_exp;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen unsigned int first, i, dest_count;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen i_assert(src_size % sizeof(*src) == 0);
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen /* @UNSAFE */
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen if (dest_count == 0) {
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen array_append(expunges, src, src_size / sizeof(*src));
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen return 0;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen }
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen src_end = CONST_PTR_OFFSET(src, src_size);
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen for (i = 0; src != src_end; src++) {
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen /* src[] must be sorted. */
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen if (src->seq1 > src->seq2 ||
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen (src+1 != src_end && src->seq2 >= src[1].seq1))
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen return -1;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen for (; i < dest_count; i++) {
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen if (src->seq1 < dest[i].seq1)
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen break;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen }
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen new_exp = *src;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen first = i;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen /* we can/must merge with next record */
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen if (new_exp.seq2 < dest[i].seq2)
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen new_exp.seq2 = dest[i].seq2;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen i++;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen }
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen /* continue previous record */
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen if (dest[first-1].seq2 < new_exp.seq2)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest[first-1].seq2 = new_exp.seq2;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen } else if (i == first) {
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen array_insert(expunges, i, &new_exp, 1);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen i++; first++;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen dest = array_get_modifiable(expunges, &dest_count);
8d131435ba4648c8821160ec38d508c97177c715Timo Sirainen } else {
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* use next record */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen dest[first] = new_exp;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen first++;
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen }
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen if (i > first) {
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen array_delete(expunges, first, i - first);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen dest = array_get_modifiable(expunges, &dest_count);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen i = first;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen }
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen }
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen return 0;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen}
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainenstatic int
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainenview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen bool *reset_r)
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen{
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen const struct mail_index_header *hdr = &view->index->map->hdr;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen uint32_t start_seq, end_seq;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen uoff_t start_offset, end_offset;
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen int ret;
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen start_seq = view->log_file_expunge_seq;
e3689d0f073341e844638f34e1e4d0b7bb053cc8Timo Sirainen start_offset = view->log_file_expunge_offset;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen end_seq = hdr->log_file_seq;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen end_offset = hdr->log_file_head_offset;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen for (;;) {
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* the view begins from the first non-synced transaction */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen ret = mail_transaction_log_view_set(view->log_view,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen start_seq, start_offset,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen end_seq, end_offset,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen reset_r);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen if (ret <= 0)
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen return ret;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen if (!*reset_r || sync_expunges)
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen break;
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen /* log was reset, but we don't want to sync expunges.
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen we can't do this, so sync only up to the reset. */
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen mail_transaction_log_view_get_prev_pos(view->log_view,
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen &end_seq, &end_offset);
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen end_seq--; end_offset = (uoff_t)-1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (end_seq < start_seq) {
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen /* we have only this reset log */
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen mail_transaction_log_view_clear(view->log_view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->log_file_expunge_seq);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen break;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen }
659fe5d24825b160cae512538088020d97a60239Timo Sirainen }
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen return 1;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen}
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainenstatic unsigned int
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenview_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen{
5cdd348121e62a6244ba2f93db781731f7129a71Timo Sirainen struct mail_index_view *view = ctx->view;
5cdd348121e62a6244ba2f93db781731f7129a71Timo Sirainen struct seq_range *src, *src_end, *dest;
5cdd348121e62a6244ba2f93db781731f7129a71Timo Sirainen unsigned int count, expunge_count = 0;
91dca97b367c54a139c268b56a0c67f564bd9197Timo Sirainen uint32_t prev_seq = 0;
659fe5d24825b160cae512538088020d97a60239Timo Sirainen
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen /* convert UIDs to sequences */
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen src = dest = array_get_modifiable(&ctx->expunges, &count);
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen src_end = src + count;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen for (; src != src_end; src++) {
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen &dest->seq1, &dest->seq2))
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen count--;
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(dest->seq1 > prev_seq);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen prev_seq = dest->seq2;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen expunge_count += dest->seq2 - dest->seq1 + 1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest++;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
659fe5d24825b160cae512538088020d97a60239Timo Sirainen array_delete(&ctx->expunges, count,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen array_count(&ctx->expunges) - count);
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen return expunge_count;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen}
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int
659fe5d24825b160cae512538088020d97a60239Timo Sirainenview_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen unsigned int *expunge_count_r)
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen{
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen struct mail_index_view *view = ctx->view;
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen const struct mail_transaction_header *hdr;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen const void *data;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen int ret;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen /* get a list of expunge transactions. there may be some that we have
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen already synced, but it doesn't matter because they'll get dropped
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen out when converting to sequences */
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen mail_transaction_log_view_mark(view->log_view);
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen &hdr, &data)) > 0) {
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen continue;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
7b3bf1de3fa7eee2185d1e404812b50c295e2b93Timo Sirainen /* this is simply a request for expunge */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen continue;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if (mail_transaction_log_sort_expunges(&ctx->expunges, data,
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen hdr->size) < 0) {
fd3f33bdb57170d63aea66ecacc8bea0f0145d6aTimo Sirainen mail_transaction_log_view_set_corrupted(view->log_view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "Corrupted expunge record");
7761758f43d6150be4b07f4c54457ce662f78c4cTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_transaction_log_view_rewind(view->log_view);
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen *expunge_count_r = view_sync_expunges2seqs(ctx);
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d39c0e195c67be5f2b0a15f25a8d6039bef02711Timo Sirainenstatic bool have_existing_expunges(struct mail_index_view *view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct seq_range *range, size_t size)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct seq_range *range_end;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t seq1, seq2;
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen range_end = CONST_PTR_OFFSET(range, size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (; range < range_end; range++) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &seq1, &seq2))
6bc98d3898c475ba7615ba2b016e5142c8b2c09fTimo Sirainen return TRUE;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen }
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen return FALSE;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen}
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic bool view_sync_have_expunges(struct mail_index_view *view)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_transaction_header *hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const void *data;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen bool have_expunges = FALSE;
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen mail_transaction_log_view_mark(view->log_view);
5626ae5e3316eced244adb6485c0927f1c7fdc41Timo Sirainen
56561d472db25ebda35ae6afdc7f7deb75c323fcTimo Sirainen while ((ret = mail_transaction_log_view_next(view->log_view,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen &hdr, &data)) > 0) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen continue;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen /* this is simply a request for expunge */
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen continue;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen }
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen /* we have an expunge. see if it still exists. */
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (have_existing_expunges(view, data, hdr->size)) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen have_expunges = TRUE;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen break;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen }
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen }
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen mail_transaction_log_view_rewind(view->log_view);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen /* handle failures as having expunges (which is safer).
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen we'll probably fail later. */
0c09e57c5d5a649c248d0073438d79acbb80c72bTimo Sirainen return ret < 0 || have_expunges;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen}
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainenstatic int uint_cmp(const void *p1, const void *p2)
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen{
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen const unsigned int *u1 = p1, *u2 = p2;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen if (*u1 < *u2)
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen return -1;
5cdd348121e62a6244ba2f93db781731f7129a71Timo Sirainen if (*u1 > *u2)
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen return 1;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen return 0;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen}
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainenstatic bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen{
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen unsigned int *old_idx, *new_idx;
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen unsigned int old_count, new_count;
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen
9bc6e10d9c6d6ffb4a2ed49a3b3d2a180f2a87a3Timo Sirainen old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
1b3bb8d39686ed24730cbc31cc9a33dc62c8c6c3Timo Sirainen new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
20c26f4fcf9ef87434761829cc209c2f84ff5716Timo Sirainen if (old_count != new_count)
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen return FALSE;
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
3776ed607821b502468bdfd5a4533af3002125d1Timo Sirainen qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen return memcmp(old_idx, new_idx, old_count * sizeof(old_idx)) == 0;
e26a771fad55dfba4d5021d12ed5685c951d9b7bTimo Sirainen}
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainenstatic int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
7c5b51bdf43a98e12c654ad437e0b258c5fffbc1Timo Sirainen uint32_t uid)
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_transaction_header thdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_transaction_keyword_update kw_up;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const unsigned int *kw_idx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *const *kw_names;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int i, count;
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen
24c6aaf3c540d078021bb4a326982ae4e3d7eaf8Timo Sirainen kw_idx = array_get(&ctx->lost_new_kw, &count);
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen if (count == 0)
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen return 0;
4c8b1c4aa0582c6ca43a4d1cbd210741e7fff952Timo Sirainen kw_names = array_idx(&ctx->view->index->keywords, 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen memset(&thdr, 0, sizeof(thdr));
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(&kw_up, 0, sizeof(kw_up));
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen kw_up.modify_type = MODIFY_ADD;
4c8b1c4aa0582c6ca43a4d1cbd210741e7fff952Timo Sirainen /* add new flags one by one */
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen for (i = 0; i < count; i++) {
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen kw_up.name_size = strlen(kw_names[kw_idx[i]]);
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen buffer_set_used_size(ctx->lost_kw_buf, 0);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen kw_up.name_size);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (ctx->lost_kw_buf->used % 4 != 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append_zero(ctx->lost_kw_buf,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen 4 - ctx->lost_kw_buf->used % 4);
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen }
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen thdr.size = ctx->lost_kw_buf->used;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen ctx->lost_kw_buf->data) < 0)
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen return -1;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen }
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen return 0;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen}
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainenstatic int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen uint32_t old_seq, uint32_t new_seq)
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen{
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen struct mail_index_map *old_map = ctx->view->map;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen struct mail_index_map *new_map = ctx->view->index->map;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen const struct mail_index_record *old_rec, *new_rec;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen struct mail_transaction_header thdr;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen const struct mail_index_ext *ext;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen const uint64_t *modseqp;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen uint64_t new_modseq;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen bool changed = FALSE;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(old_map, old_seq - 1);
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen new_rec = MAIL_INDEX_MAP_IDX(new_map, new_seq - 1);
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen memset(&thdr, 0, sizeof(thdr));
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if (old_rec->flags != new_rec->flags) {
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen struct mail_transaction_flag_update flag_update;
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen /* check this before syncing the record, since it updates
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen old_rec. */
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen (new_rec->flags & MAIL_INDEX_FLAGS_MASK))
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen changed = TRUE;
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen MAIL_TRANSACTION_EXTERNAL;
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen thdr.size = sizeof(flag_update);
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen memset(&flag_update, 0, sizeof(flag_update));
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen flag_update.uid1 = flag_update.uid2 = new_rec->uid;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen flag_update.add_flags = new_rec->flags;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen flag_update.remove_flags = ~new_rec->flags & 0xff;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen &flag_update) < 0)
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen return -1;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen }
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw);
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!view_sync_lost_keywords_equal(ctx)) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen struct mail_transaction_keyword_reset kw_reset;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen thdr.type = MAIL_TRANSACTION_KEYWORD_RESET |
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen MAIL_TRANSACTION_EXTERNAL;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen thdr.size = sizeof(kw_reset);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen /* remove all old flags by resetting them */
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen memset(&kw_reset, 0, sizeof(kw_reset));
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen kw_reset.uid1 = kw_reset.uid2 = new_rec->uid;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen &kw_reset) < 0)
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen view_sync_update_keywords(ctx, new_rec->uid);
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen changed = TRUE;
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen }
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen if (changed) {
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen /* flags or keywords changed */
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen } else if (ctx->view->highest_modseq != 0 &&
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen ctx->lost_new_ext_idx != (uint32_t)-1) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen /* if modseq has changed include this message in changed flags
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen list, even if we didn't see any changes above. */
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen new_modseq = *modseqp;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen if (new_modseq > ctx->view->highest_modseq)
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen changed = TRUE;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen }
23230b2a3fe09ed29bd1314c7b8b4a0b17ea0ecfTimo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen /* without modseqs lost_flags isn't updated perfectly correctly, because
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen by the time we're comparing old flags it may have changed from what
0b924289a76fadafc3fdabcbc70e6f5562d230b6Timo Sirainen we last sent to the client (because the map is shared). This could
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen be avoided by always keeping a private copy of the map in the view,
bbe90c4805e7032fdcefde3df125df0ca8be1befTimo Sirainen but that's a waste of memory for as rare of a problem as this. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (changed)
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen seq_range_array_add(&ctx->lost_flags, 0, new_rec->uid);
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen return 0;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen}
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainenview_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen unsigned int *expunge_count_r)
24c6aaf3c540d078021bb4a326982ae4e3d7eaf8Timo Sirainen{
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen struct mail_index_view *view = ctx->view;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen struct mail_index_map *old_map = view->map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_map *new_map = view->index->map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const unsigned int old_count = old_map->hdr.messages_count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const unsigned int new_count = new_map->hdr.messages_count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_index_record *old_rec, *new_rec;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen struct mail_transaction_header thdr;
88553367d677170a4b703b9d52aac9eabf91c656Timo Sirainen unsigned int i, j;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen /* we don't update the map in the same order as it's typically done.
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen map->rec_map may already have some messages appended that we don't
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen want. get an atomic map to make sure these get removed. */
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen (void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen &ctx->lost_new_ext_idx))
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen ctx->lost_new_ext_idx = (uint32_t)-1;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen i_array_init(&ctx->lost_flags, 64);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen t_array_init(&ctx->lost_old_kw, 32);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen t_array_init(&ctx->lost_new_kw, 32);
659fe5d24825b160cae512538088020d97a60239Timo Sirainen ctx->lost_kw_buf = buffer_create_dynamic(pool_datastack_create(), 128);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen /* handle expunges and sync flags */
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen i = j = 0;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen while (i < old_count && j < new_count) {
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen if (old_rec->uid == new_rec->uid) {
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen /* message found - check if flags have changed */
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen if (view_sync_apply_lost_changes(ctx, i + 1, j + 1) < 0)
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen return -1;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen i++; j++;
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen } else if (old_rec->uid < new_rec->uid) {
e063aca6bc2f08bec516d4b631052ea9191f011dTimo Sirainen /* message expunged */
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen i++;
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen } else {
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen /* new message appeared out of nowhere */
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen mail_index_set_error(view->index,
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen "%s view is inconsistent: "
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen "uid=%u inserted in the middle of mailbox",
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen view->index->filepath, new_rec->uid);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen return -1;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen /* if there are old messages left, they're all expunged */
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen for (; i < old_count; i++) {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen old_rec = MAIL_INDEX_MAP_IDX(old_map, i);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen seq_range_array_add(&ctx->expunges, 0, old_rec->uid);
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen }
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen /* if there are new messages left, they're all new messages */
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen thdr.size = sizeof(*new_rec);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen for (; j < new_count; j++) {
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen new_rec = MAIL_INDEX_MAP_IDX(new_map, j);
d0143523a87b41eae0b118ff03aad539903b3555Timo Sirainen if (mail_index_sync_record(&ctx->sync_map_ctx,
d0143523a87b41eae0b118ff03aad539903b3555Timo Sirainen &thdr, new_rec) < 0)
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen return -1;
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen mail_index_map_lookup_keywords(new_map, j + 1,
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen &ctx->lost_new_kw);
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen view_sync_update_keywords(ctx, new_rec->uid);
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen }
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen *expunge_count_r = view_sync_expunges2seqs(ctx);
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen /* we have no idea how far we've synced - make sure these aren't used */
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen old_map->hdr.log_file_seq = 0;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen old_map->hdr.log_file_head_offset = 0;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen old_map->hdr.log_file_tail_offset = 0;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen array_clear(&ctx->expunges);
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen ctx->skipped_expunges = *expunge_count_r > 0;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen } else {
faed8babca9914257f34fb2e603d74016d563b2dTimo Sirainen view->log_file_head_seq = new_map->hdr.log_file_seq;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen view->log_file_head_offset = new_map->hdr.log_file_head_offset;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen }
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen return 0;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen}
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainenstatic int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx)
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen{
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen struct mail_index_view *view = ctx->view;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen uint32_t seq;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen uoff_t offset;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen bool reset;
16c89b1260c9d07c01c83a9219424d3727069b2eTimo Sirainen
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen /* replace the view's map */
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen view->index->map->refcount++;
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen mail_index_unmap(&view->map);
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen view->map = view->index->map;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen /* update log positions */
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen view->log_file_head_seq = seq = view->map->hdr.log_file_seq;
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen view->log_file_head_offset = offset =
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen view->map->hdr.log_file_head_offset;
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen
26ff8f8a4867bf8e9551a27a2de8c12cd138b065Timo Sirainen if (mail_transaction_log_view_set(view->log_view, seq, offset,
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen seq, offset, &reset) < 0)
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen return -1;
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen view->inconsistent = FALSE;
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen return 0;
8ef80b0b9c73fb0a0188788b14b3e15084b7a452Timo Sirainen}
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainen
d67f54632110cfb6aafe2d7cd1f99b031c0b208aTimo Sirainenstruct mail_index_view_sync_ctx *
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_index_view_sync_begin(struct mail_index_view *view,
020a39a395d2adb768e0179631b37bc78ecd9471Timo Sirainen enum mail_index_view_sync_flags flags)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_view_sync_ctx *ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_map *map;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen unsigned int expunge_count = 0;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen bool reset, sync_expunges, have_expunges;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen int ret;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen i_assert(!view->syncing);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen i_assert(view->transactions == 0);
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->syncing = TRUE;
b35f7104715edee0cfac6d46ab0b342033867eb7Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* Syncing the view invalidates all previous looked up records.
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen Unreference the mappings this view keeps because of them. */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_view_unref_maps(view);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen ctx = i_new(struct mail_index_view_sync_ctx, 1);
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen ctx->view = view;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->flags = flags;
94a8cb0ee1d85569ad1a2acacd92d3ce22f8a1cbTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (sync_expunges)
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen i_array_init(&ctx->expunges, 64);
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen /* just get this view synced - don't return anything */
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen i_assert(sync_expunges);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (mail_index_view_sync_init_fix(ctx) < 0)
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen ctx->failed = TRUE;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen return ctx;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen }
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (mail_index_view_is_inconsistent(view)) {
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen mail_index_set_error(view->index, "%s view is inconsistent",
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen view->index->filepath);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen ctx->failed = TRUE;
cc6ed00c61fda24799c905e403b94a2a8c39ae5cTimo Sirainen return ctx;
cc6ed00c61fda24799c905e403b94a2a8c39ae5cTimo Sirainen }
cc6ed00c61fda24799c905e403b94a2a8c39ae5cTimo Sirainen
cc6ed00c61fda24799c905e403b94a2a8c39ae5cTimo Sirainen ret = view_sync_set_log_view_range(view, sync_expunges, &reset);
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (ret < 0) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen ctx->failed = TRUE;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen return ctx;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen }
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen mail_index_sync_map_init(&ctx->sync_map_ctx, view,
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen MAIL_INDEX_SYNC_HANDLER_VIEW);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret == 0) {
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen ctx->log_was_lost = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!sync_expunges)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_array_init(&ctx->expunges, 64);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (view_sync_get_log_lost_changes(ctx, &expunge_count) < 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_set_error(view->index,
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen "%s view syncing failed to apply changes",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->index->filepath);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen view->inconsistent = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->failed = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen have_expunges = expunge_count > 0;
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen } else if (sync_expunges) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen /* get list of all expunges first */
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen if (view_sync_get_expunges(ctx, &expunge_count) < 0) {
8afec4d1a32b78f540257a27769b372aad753384Timo Sirainen ctx->failed = TRUE;
e06c0b65c16ccce69bbee009ead14d7d3d17a256Timo Sirainen return ctx;
23230b2a3fe09ed29bd1314c7b8b4a0b17ea0ecfTimo Sirainen }
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen have_expunges = expunge_count > 0;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen } else {
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen have_expunges = view_sync_have_expunges(view);
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen }
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
23230b2a3fe09ed29bd1314c7b8b4a0b17ea0ecfTimo Sirainen ctx->finish_min_msg_count = reset ? 0 :
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen view->map->hdr.messages_count - expunge_count;
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if (reset && view->map->hdr.messages_count > 0) {
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen view->inconsistent = TRUE;
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen mail_index_set_error(view->index,
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen "%s reset, view is now inconsistent",
23230b2a3fe09ed29bd1314c7b8b4a0b17ea0ecfTimo Sirainen view->index->filepath);
5a58037ad75b88356d82240fab2bc604de03107eTimo Sirainen }
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen if (!have_expunges) {
0d70a702dec63d22535684fec6a7247c5f153208Timo Sirainen /* no expunges, we can just replace the map */
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen if (view->index->map->hdr.messages_count <
3776ed607821b502468bdfd5a4533af3002125d1Timo Sirainen ctx->finish_min_msg_count) {
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen mail_index_set_error(view->index,
945631faab2bf1aed8d95a1fd0c317a9ce153725Timo Sirainen "Index %s lost messages without expunging "
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen "(%u -> %u)", view->index->filepath,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen view->map->hdr.messages_count,
7888a9d2008eab9985096c46e1da9ee985c22a2aTimo Sirainen view->index->map->hdr.messages_count);
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen ctx->finish_min_msg_count = 0;
b032dc80e358f09893f09999f172ff12f5dbbb8eTimo Sirainen view->inconsistent = TRUE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->index->map->refcount++;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_unmap(&view->map);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view->map = view->index->map;
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen } else {
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen /* expunges seen. create a private map which we update.
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen if we're syncing expunges the map will finally be replaced
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen with the head map to remove the expunged messages. */
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen ctx->sync_map_update = TRUE;
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen if (view->map->refcount > 1) {
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen map = mail_index_map_clone(view->map);
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen mail_index_unmap(&view->map);
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen view->map = map;
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen } else {
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen map = view->map;
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen }
87ca2e468841829b44c09d618ac02f61a30b7a49Timo Sirainen
03739a8eaad2d8b34b9d87dbbe5b13c5d5dfa11aTimo Sirainen if (sync_expunges) {
ctx->sync_new_map = view->index->map;
ctx->sync_new_map->refcount++;
}
}
#ifdef DEBUG
mail_index_map_check(view->map);
#endif
return ctx;
}
static bool
view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
{
const struct mail_index_view_log_sync_area *syncs;
unsigned int i, count;
if (!array_is_created(&view->syncs_hidden))
return FALSE;
syncs = array_get(&view->syncs_hidden, &count);
for (i = 0; i < count; i++) {
if (syncs[i].log_file_offset <= offset &&
offset - syncs[i].log_file_offset < syncs[i].length &&
syncs[i].log_file_seq == seq)
return TRUE;
}
return FALSE;
}
static bool
mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
const struct mail_transaction_header *hdr)
{
struct mail_index_view *view = ctx->view;
uint32_t seq;
uoff_t offset, next_offset;
mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
next_offset = offset + sizeof(*hdr) + hdr->size;
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
(hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
i_assert(!LOG_IS_BEFORE(seq, offset,
view->log_file_expunge_seq,
view->log_file_expunge_offset));
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = seq;
view->log_file_expunge_offset = offset;
ctx->skipped_expunges = TRUE;
}
return FALSE;
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
view->log_file_expunge_offset)) {
/* already synced */
return FALSE;
}
}
if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
view->log_file_head_offset)) {
/* already synced */
return FALSE;
}
view->log_file_head_seq = seq;
view->log_file_head_offset = next_offset;
return TRUE;
}
static int
mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
{
struct mail_transaction_log_view *log_view = ctx->view->log_view;
struct mail_index_view *view = ctx->view;
const struct mail_transaction_header *hdr;
uint32_t seq;
uoff_t offset;
int ret;
bool synced_to_map;
do {
/* Get the next transaction from log. */
ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
&ctx->data);
if (ret <= 0) {
if (ret < 0)
return -1;
ctx->hdr = NULL;
ctx->last_read = TRUE;
return 0;
}
hdr = ctx->hdr;
/* skip records we've already synced */
} while (!mail_index_view_sync_want(ctx, hdr));
mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
/* If we started from a map that we didn't create ourself,
some of the transactions may already be synced. at the end
of this view sync we'll update file_seq=0 so that this check
always becomes FALSE for subsequent syncs. */
synced_to_map = view->map->hdr.log_file_seq != 0 &&
LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset);
/* Apply transaction to view's mapping if needed (meaning we
didn't just re-map the view to head mapping). */
if (ctx->sync_map_update && !synced_to_map) {
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0) T_BEGIN {
ret = mail_index_sync_record(&ctx->sync_map_ctx,
hdr, ctx->data);
} T_END;
if (ret < 0)
return -1;
}
ctx->hidden = view_sync_is_hidden(view, seq, offset);
return 1;
}
static bool
mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *rec)
{
const struct mail_transaction_header *hdr = ctx->hdr;
const void *data = ctx->data;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_flag_update[] */
for (;;) {
ctx->data_offset += sizeof(*update);
if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update))
break;
/* skip internal flag changes */
if (ctx->data_offset == ctx->hdr->size)
return 0;
update = CONST_PTR_OFFSET(data, ctx->data_offset);
}
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = update->uid1;
rec->uid2 = update->uid2;
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *update = data;
const uint32_t *uids;
/* data contains mail_transaction_keyword_update header,
the keyword name and an array of { uint32_t uid1, uid2; } */
if (ctx->data_offset == 0) {
/* skip over the header and name */
ctx->data_offset = sizeof(*update) + update->name_size;
if ((ctx->data_offset % 4) != 0)
ctx->data_offset += 4 - (ctx->data_offset % 4);
}
uids = CONST_PTR_OFFSET(data, ctx->data_offset);
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = uids[0];
rec->uid2 = uids[1];
ctx->data_offset += sizeof(uint32_t) * 2;
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *reset =
CONST_PTR_OFFSET(data, ctx->data_offset);
/* data contains mail_transaction_keyword_reset[] */
rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
rec->uid1 = reset->uid1;
rec->uid2 = reset->uid2;
ctx->data_offset += sizeof(*reset);
break;
}
default:
ctx->hdr = NULL;
return FALSE;
}
rec->hidden = ctx->hidden;
return TRUE;
}
static bool
mail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
const struct seq_range *range;
unsigned int count;
range = array_get(&ctx->lost_flags, &count);
if (ctx->lost_flag_idx == count) {
ctx->last_read = TRUE;
return FALSE;
}
sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
sync_rec->uid1 = range[ctx->lost_flag_idx].seq1;
sync_rec->uid2 = range[ctx->lost_flag_idx].seq2;
ctx->lost_flag_idx++;
return TRUE;
}
bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_view_sync_rec *sync_rec)
{
int ret;
if (ctx->log_was_lost)
return mail_index_view_sync_next_lost(ctx, sync_rec);
do {
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ret = mail_index_view_sync_get_next_transaction(ctx);
if (ret <= 0) {
if (ret < 0)
ctx->failed = TRUE;
return FALSE;
}
ctx->data_offset = 0;
}
} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
return TRUE;
}
void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
const ARRAY_TYPE(seq_range) **expunges_r)
{
*expunges_r = &ctx->expunges;
}
static void
mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
{
const struct mail_index_view_log_sync_area *syncs;
unsigned int i, count;
if (!array_is_created(&view->syncs_hidden))
return;
/* Clean up to view's tail */
syncs = array_get(&view->syncs_hidden, &count);
for (i = 0; i < count; i++) {
if ((syncs[i].log_file_offset +
syncs[i].length > view->log_file_expunge_offset &&
syncs[i].log_file_seq == view->log_file_expunge_seq) ||
syncs[i].log_file_seq > view->log_file_expunge_seq)
break;
}
if (i > 0)
array_delete(&view->syncs_hidden, 0, i);
}
int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
bool *delayed_expunges_r)
{
struct mail_index_view_sync_ctx *ctx = *_ctx;
struct mail_index_view *view = ctx->view;
int ret = ctx->failed ? -1 : 0;
i_assert(view->syncing);
*_ctx = NULL;
*delayed_expunges_r = ctx->skipped_expunges;
if ((!ctx->last_read || view->inconsistent) &&
(ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
/* we didn't sync everything */
view->inconsistent = TRUE;
ret = -1;
}
if (ctx->sync_map_ctx.view != NULL)
mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
if (ctx->sync_new_map != NULL) {
mail_index_unmap(&view->map);
view->map = ctx->sync_new_map;
} else if (ctx->sync_map_update) {
/* log offsets have no meaning in views. make sure they're not
tried to be used wrong by setting them to zero. */
view->map->hdr.log_file_seq = 0;
view->map->hdr.log_file_head_offset = 0;
view->map->hdr.log_file_tail_offset = 0;
}
i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
if (!ctx->skipped_expunges) {
view->log_file_expunge_seq = view->log_file_head_seq;
view->log_file_expunge_offset = view->log_file_head_offset;
}
if (ctx->sync_map_ctx.view != NULL)
mail_index_sync_map_deinit(&ctx->sync_map_ctx);
mail_index_view_sync_clean_log_syncs(ctx->view);
#ifdef DEBUG
mail_index_map_check(view->map);
#endif
/* set log view to empty range so unneeded memory gets freed */
mail_transaction_log_view_clear(view->log_view,
view->log_file_expunge_seq);
if (array_is_created(&ctx->expunges))
array_free(&ctx->expunges);
if (array_is_created(&ctx->lost_flags))
array_free(&ctx->lost_flags);
view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
view->syncing = FALSE;
i_free(ctx);
return ret;
}
void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset,
unsigned int length)
{
struct mail_index_view_log_sync_area *area;
if (!array_is_created(&view->syncs_hidden))
i_array_init(&view->syncs_hidden, 32);
area = array_append_space(&view->syncs_hidden);
area->log_file_seq = log_file_seq;
area->log_file_offset = log_file_offset;
area->length = length;
}