970N/A

970N/A#include "lib.h"

970N/A#include "array.h"

1500N/A#include "mail-index-view-private.h"

970N/A#include "mail-index-sync-private.h"

970N/A#include "mail-index-transaction-private.h"

970N/A#include "mail-transaction-log-private.h"

970N/A#include "mail-cache.h"

970N/A

970N/A#include <stdio.h>

970N/A#include <stdlib.h>

970N/A

970N/Astruct mail_index_sync_ctx {

970N/A struct mail_index *index;

970N/A struct mail_index_view *view;

970N/A struct mail_index_transaction *sync_trans, *ext_trans;

970N/A enum mail_index_sync_flags flags;

970N/A

970N/A const struct mail_transaction_header *hdr;

970N/A const void *data;

970N/A

970N/A ARRAY_DEFINE(sync_list, struct mail_index_sync_list);

970N/A uint32_t next_uid;

970N/A uint32_t last_tail_seq, last_tail_offset;

970N/A

970N/A uint32_t append_uid_first, append_uid_last;

970N/A

970N/A unsigned int sync_appends:1;

970N/A};

970N/A

970N/Astatic void mail_index_sync_add_expunge(struct mail_index_sync_ctx *ctx)

970N/A{

970N/A const struct mail_transaction_expunge *e = ctx->data;

970N/A size_t i, size = ctx->hdr->size / sizeof(*e);

970N/A uint32_t uid;

970N/A

970N/A for (i = 0; i < size; i++) {

970N/A for (uid = e[i].uid1; uid <= e[i].uid2; uid++)

970N/A mail_index_expunge(ctx->sync_trans, uid);

970N/A }

970N/A}

970N/A

1105N/Astatic void mail_index_sync_add_flag_update(struct mail_index_sync_ctx *ctx)

970N/A{

970N/A const struct mail_transaction_flag_update *u = ctx->data;

970N/A size_t i, size = ctx->hdr->size / sizeof(*u);

970N/A

970N/A for (i = 0; i < size; i++) {

970N/A if (u[i].add_flags != 0) {

970N/A mail_index_update_flags_range(ctx->sync_trans,

970N/A u[i].uid1, u[i].uid2,

970N/A MODIFY_ADD,

970N/A u[i].add_flags);

970N/A }

970N/A if (u[i].remove_flags != 0) {

970N/A mail_index_update_flags_range(ctx->sync_trans,

970N/A u[i].uid1, u[i].uid2,

970N/A MODIFY_REMOVE,

970N/A u[i].remove_flags);

970N/A }

970N/A }

970N/A}

970N/A

970N/Astatic void mail_index_sync_add_keyword_update(struct mail_index_sync_ctx *ctx)

970N/A{

970N/A const struct mail_transaction_keyword_update *u = ctx->data;

970N/A const char *keyword_names[2];

970N/A struct mail_keywords *keywords;

1130N/A const uint32_t *uids;

970N/A uint32_t uid;

970N/A size_t uidset_offset, i, size;

970N/A

970N/A uidset_offset = sizeof(*u) + u->name_size;

970N/A if ((uidset_offset % 4) != 0)

970N/A uidset_offset += 4 - (uidset_offset % 4);

970N/A uids = CONST_PTR_OFFSET(u, uidset_offset);

970N/A

970N/A keyword_names[0] = t_strndup(u + 1, u->name_size);

970N/A keyword_names[1] = NULL;

970N/A keywords = mail_index_keywords_create(ctx->index, keyword_names);

970N/A

970N/A size = (ctx->hdr->size - uidset_offset) / sizeof(uint32_t);

970N/A for (i = 0; i < size; i += 2) {

970N/A /* FIXME: mail_index_update_keywords_range() */

970N/A for (uid = uids[i]; uid <= uids[i+1]; uid++) {

970N/A mail_index_update_keywords(ctx->sync_trans, uid,

970N/A u->modify_type, keywords);

970N/A }

970N/A }

970N/A

970N/A mail_index_keywords_free(&keywords);

970N/A}

970N/A

970N/Astatic void mail_index_sync_add_keyword_reset(struct mail_index_sync_ctx *ctx)

970N/A{

970N/A const struct mail_transaction_keyword_reset *u = ctx->data;

970N/A size_t i, size = ctx->hdr->size / sizeof(*u);

1105N/A struct mail_keywords *keywords;

1105N/A uint32_t uid;

970N/A

1105N/A keywords = mail_index_keywords_create(ctx->index, NULL);

970N/A for (i = 0; i < size; i++) {

970N/A for (uid = u[i].uid1; uid <= u[i].uid2; uid++) {

970N/A mail_index_update_keywords(ctx->sync_trans, uid,

970N/A MODIFY_REPLACE, keywords);

970N/A }

970N/A }

1154N/A mail_index_keywords_free(&keywords);

1152N/A}

970N/A

1172N/Astatic void mail_index_sync_add_append(struct mail_index_sync_ctx *ctx)

1172N/A{

1172N/A const struct mail_index_record *rec = ctx->data;

970N/A

1105N/A if (ctx->append_uid_first == 0 || rec->uid < ctx->append_uid_first)

970N/A ctx->append_uid_first = rec->uid;

970N/A

1105N/A rec = CONST_PTR_OFFSET(ctx->data, ctx->hdr->size - sizeof(*rec));

970N/A if (rec->uid > ctx->append_uid_last)

970N/A ctx->append_uid_last = rec->uid;

1120N/A

1120N/A ctx->sync_appends = TRUE;

1120N/A}

1479N/A

1479N/Astatic bool mail_index_sync_add_transaction(struct mail_index_sync_ctx *ctx)

1172N/A{

1172N/A switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) {

1153N/A case MAIL_TRANSACTION_EXPUNGE:

970N/A mail_index_sync_add_expunge(ctx);

970N/A break;

1356N/A case MAIL_TRANSACTION_FLAG_UPDATE:

1356N/A mail_index_sync_add_flag_update(ctx);

1356N/A break;

1356N/A case MAIL_TRANSACTION_KEYWORD_UPDATE:

1356N/A mail_index_sync_add_keyword_update(ctx);

1356N/A break;

1356N/A case MAIL_TRANSACTION_KEYWORD_RESET:

1356N/A mail_index_sync_add_keyword_reset(ctx);

1356N/A break;

1356N/A case MAIL_TRANSACTION_APPEND:

970N/A mail_index_sync_add_append(ctx);

970N/A break;

1105N/A default:

1105N/A return FALSE;

1105N/A }

970N/A return TRUE;

970N/A}

970N/A

1479N/Astatic void mail_index_sync_add_dirty_updates(struct mail_index_sync_ctx *ctx)

1479N/A{

1105N/A struct mail_transaction_flag_update update;

970N/A const struct mail_index_record *rec;

970N/A uint32_t seq, messages_count;

1479N/A

1479N/A memset(&update, 0, sizeof(update));

970N/A

970N/A messages_count = mail_index_view_get_messages_count(ctx->view);

970N/A for (seq = 1; seq <= messages_count; seq++) {

970N/A rec = mail_index_lookup(ctx->view, seq);

970N/A if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0)

970N/A continue;

970N/A

1316N/A mail_index_update_flags(ctx->sync_trans, rec->uid,

1189N/A MODIFY_REPLACE, rec->flags);

1189N/A }

1189N/A}

1316N/A

1189N/Astatic void

1316N/Amail_index_sync_update_mailbox_pos(struct mail_index_sync_ctx *ctx)

1189N/A{

1189N/A uint32_t seq;

1189N/A uoff_t offset;

970N/A

970N/A mail_transaction_log_view_get_prev_pos(ctx->view->log_view,

970N/A &seq, &offset);

970N/A

970N/A ctx->last_tail_seq = seq;

1189N/A ctx->last_tail_offset = offset + ctx->hdr->size + sizeof(*ctx->hdr);

1189N/A}

970N/A

970N/Astatic int

970N/Amail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx)

1132N/A{

1132N/A struct mail_index_transaction *sync_trans = ctx->sync_trans;

1132N/A struct mail_index_sync_list *synclist;

970N/A const struct mail_index_transaction_keyword_update *keyword_updates;

970N/A unsigned int i, keyword_count;

970N/A int ret;

970N/A

970N/A if ((ctx->view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) &&

1276N/A (ctx->flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0) {

1316N/A /* show dirty flags as flag updates */

970N/A mail_index_sync_add_dirty_updates(ctx);

970N/A }

1132N/A

1132N/A /* read all transactions from log into a transaction in memory.

1132N/A while ((ret = mail_transaction_log_view_next(ctx->view->log_view,

970N/A &ctx->hdr,

1132N/A &ctx->data)) > 0) {

1132N/A if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)

1132N/A continue;

1132N/A

1132N/A T_BEGIN {

1147N/A if (mail_index_sync_add_transaction(ctx))

1147N/A mail_index_sync_update_mailbox_pos(ctx);

1147N/A } T_END;

1147N/A }

1132N/A

1132N/A /* create an array containing all expunge, flag and keyword update

1132N/A keyword_count = !array_is_created(&sync_trans->keyword_updates) ? 0 :

1132N/A array_count(&sync_trans->keyword_updates);

1132N/A i_array_init(&ctx->sync_list, keyword_count + 2);

1298N/A

1298N/A if (array_is_created(&sync_trans->expunges)) {

1298N/A synclist = array_append_space(&ctx->sync_list);

1298N/A synclist->array = (void *)&sync_trans->expunges;

1479N/A }

1479N/A

1479N/A if (array_is_created(&sync_trans->updates)) {

1298N/A synclist = array_append_space(&ctx->sync_list);

1153N/A synclist->array = (void *)&sync_trans->updates;

1153N/A }

1153N/A

1153N/A /* we must return resets before keyword additions or they get lost */

1172N/A if (array_is_created(&sync_trans->keyword_resets)) {

970N/A synclist = array_append_space(&ctx->sync_list);

970N/A synclist->array = (void *)&sync_trans->keyword_resets;

1003N/A }

1003N/A

1392N/A keyword_updates = keyword_count == 0 ? NULL :

970N/A array_idx(&sync_trans->keyword_updates, 0);

970N/A for (i = 0; i < keyword_count; i++) {

1153N/A if (array_is_created(&keyword_updates[i].add_seq)) {

1172N/A synclist = array_append_space(&ctx->sync_list);

1172N/A synclist->array = (void *)&keyword_updates[i].add_seq;

970N/A synclist->keyword_idx = i;

1172N/A }

1154N/A if (array_is_created(&keyword_updates[i].remove_seq)) {

970N/A synclist = array_append_space(&ctx->sync_list);

970N/A synclist->array =

970N/A (void *)&keyword_updates[i].remove_seq;

970N/A synclist->keyword_idx = i;

970N/A synclist->keyword_remove = TRUE;

970N/A }

970N/A }

970N/A

1105N/A return ret;

1105N/A}

970N/A

1105N/Astatic bool

1105N/Amail_index_need_sync(struct mail_index *index, enum mail_index_sync_flags flags,

1153N/A uint32_t log_file_seq, uoff_t log_file_offset)

1153N/A{

970N/A const struct mail_index_header *hdr = &index->map->hdr;

1153N/A if ((flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) == 0)

1153N/A return TRUE;

1153N/A

1153N/A /* sync only if there's something to do */

970N/A if (hdr->first_recent_uid < hdr->next_uid &&

1479N/A (flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0)

1479N/A return TRUE;

1479N/A

1479N/A if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) &&

1479N/A (flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0)

1479N/A return TRUE;

1479N/A

1479N/A if (log_file_seq == (uint32_t)-1) {

970N/A /* we want to sync up to transaction log's head */

1261N/A mail_transaction_log_get_head(index->log,

1261N/A &log_file_seq, &log_file_offset);

1261N/A }

1261N/A if ((hdr->log_file_tail_offset < log_file_offset &&

1261N/A hdr->log_file_seq == log_file_seq) ||

970N/A hdr->log_file_seq < log_file_seq)

1261N/A return TRUE;

1261N/A

1152N/A if (index->need_recreate)

1261N/A return TRUE;

1152N/A

970N/A /* already synced */

970N/A return mail_cache_need_compress(index->cache);

970N/A}

970N/A

970N/Astatic int

970N/Amail_index_sync_set_log_view(struct mail_index_view *view,

970N/A uint32_t start_file_seq, uoff_t start_file_offset)

970N/A{

970N/A uint32_t log_seq;

970N/A uoff_t log_offset;

970N/A bool reset;

970N/A int ret;

970N/A

970N/A mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset);

970N/A

970N/A ret = mail_transaction_log_view_set(view->log_view,

970N/A start_file_seq, start_file_offset,

1153N/A log_seq, log_offset, &reset);

970N/A if (ret <= 0) {

970N/A /* either corrupted or the file was deleted for

970N/A mail_index_set_error(view->index,

970N/A "Unexpected transaction log desync with index %s",

970N/A view->index->filepath);

1152N/A return -1;

970N/A }

970N/A return 0;

970N/A}

970N/A

970N/Aint mail_index_sync_begin(struct mail_index *index,

1261N/A struct mail_index_sync_ctx **ctx_r,

1261N/A struct mail_index_view **view_r,

1261N/A struct mail_index_transaction **trans_r,

1261N/A enum mail_index_sync_flags flags)

970N/A{

970N/A int ret;

970N/A

970N/A ret = mail_index_sync_begin_to(index, ctx_r, view_r, trans_r,

970N/A (uint32_t)-1, (uoff_t)-1, flags);

1046N/A i_assert(ret != 0 ||

1046N/A (flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) != 0);

1046N/A return ret;

1046N/A}

1261N/A

1261N/Astatic int

1261N/Amail_index_sync_begin_init(struct mail_index *index,

1261N/A enum mail_index_sync_flags flags,

1261N/A uint32_t log_file_seq, uoff_t log_file_offset)

1261N/A{

1261N/A const struct mail_index_header *hdr;

1261N/A uint32_t seq;

1261N/A uoff_t offset;

1261N/A bool locked = FALSE;

1152N/A int ret;

1152N/A

1261N/A /* if we require changes, don't lock transaction log yet. first check

970N/A if ((flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) == 0) {

1152N/A if (mail_transaction_log_sync_lock(index->log,

1152N/A &seq, &offset) < 0)

970N/A return -1;

970N/A locked = TRUE;

970N/A }

1261N/A

1261N/A /* The view must contain what we expect the mailbox to look like

1152N/A if ((ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD)) <= 0) {

1152N/A if (ret == 0) {

1152N/A if (locked)

1152N/A mail_transaction_log_sync_unlock(index->log);

1152N/A return -1;

1152N/A }

1153N/A

1153N/A /* let's try again */

1153N/A if (mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) {

1153N/A if (locked)

1153N/A mail_transaction_log_sync_unlock(index->log);

1153N/A return -1;

1152N/A }

1153N/A }

1152N/A

1152N/A if (!mail_index_need_sync(index, flags,

1153N/A log_file_seq, log_file_offset)) {

1152N/A if (locked)

1152N/A mail_transaction_log_sync_unlock(index->log);

1152N/A return 0;

970N/A }

1153N/A

1153N/A if (!locked) {

1153N/A /* it looks like we have something to sync. lock the file and

1153N/A flags &= ~MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;

1153N/A return mail_index_sync_begin_init(index, flags, log_file_seq,

1153N/A log_file_offset);

1153N/A }

1153N/A

1130N/A hdr = &index->map->hdr;

1130N/A if (hdr->log_file_tail_offset > hdr->log_file_head_offset ||

1130N/A hdr->log_file_seq > seq ||

1130N/A (hdr->log_file_seq == seq && hdr->log_file_tail_offset > offset)) {

1130N/A /* broken sync positions. fix them. */

1130N/A mail_index_set_error(index,

1130N/A "broken sync positions in index file %s",

1130N/A index->filepath);

1130N/A mail_index_fsck_locked(index);

1130N/A }

1130N/A return 1;

1130N/A}

1130N/A

1130N/Aint mail_index_sync_begin_to(struct mail_index *index,

1130N/A struct mail_index_sync_ctx **ctx_r,

1161N/A struct mail_index_view **view_r,

1130N/A struct mail_index_transaction **trans_r,

1130N/A uint32_t log_file_seq, uoff_t log_file_offset,

1130N/A enum mail_index_sync_flags flags)

1130N/A{

1130N/A const struct mail_index_header *hdr;

1130N/A struct mail_index_sync_ctx *ctx;

1130N/A struct mail_index_view *sync_view;

1172N/A enum mail_index_transaction_flags trans_flags;

1130N/A int ret;

1130N/A

1130N/A i_assert(!index->syncing);

1130N/A

1130N/A if (log_file_seq != (uint32_t)-1)

1161N/A flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;

1130N/A

970N/A ret = mail_index_sync_begin_init(index, flags, log_file_seq,

970N/A log_file_offset);

970N/A if (ret <= 0)

1161N/A return ret;

970N/A

970N/A hdr = &index->map->hdr;

970N/A

970N/A ctx = i_new(struct mail_index_sync_ctx, 1);

970N/A ctx->index = index;

1139N/A ctx->last_tail_seq = hdr->log_file_seq;

1139N/A ctx->last_tail_offset = hdr->log_file_tail_offset;

970N/A ctx->flags = flags;

970N/A

970N/A ctx->view = mail_index_view_open(index);

970N/A

970N/A sync_view = mail_index_dummy_view_open(index);

1356N/A ctx->sync_trans = mail_index_transaction_begin(sync_view,

1356N/A MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);

1130N/A mail_index_view_close(&sync_view);

970N/A

1130N/A /* we wish to see all the changes from last mailbox sync position to

1356N/A if (mail_index_sync_set_log_view(ctx->view, hdr->log_file_seq,

970N/A hdr->log_file_tail_offset) < 0) {

970N/A /* if a log file is missing, there's nothing we can do except

1356N/A mail_index_fsck_locked(index);

1132N/A mail_index_sync_rollback(&ctx);

970N/A return mail_index_sync_begin_to(index, ctx_r, view_r, trans_r,

970N/A log_file_seq, log_file_offset,

970N/A flags);

970N/A }

1356N/A

1356N/A /* we need to have all the transactions sorted to optimize

970N/A if (mail_index_sync_read_and_sort(ctx) < 0) {

970N/A mail_index_sync_rollback(&ctx);

970N/A return -1;

970N/A }

970N/A

970N/A ctx->view->index_sync_view = TRUE;

1139N/A

970N/A /* create the transaction after the view has been updated with

1130N/A trans_flags = MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;

1130N/A if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES) != 0)

1130N/A trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES;

1130N/A if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_FSYNC) != 0)

1130N/A trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_FSYNC;

1130N/A ctx->ext_trans = mail_index_transaction_begin(ctx->view, trans_flags);

1130N/A ctx->ext_trans->sync_transaction = TRUE;

1130N/A

1130N/A index->syncing = TRUE;

1130N/A

1130N/A *ctx_r = ctx;

1130N/A *view_r = ctx->view;

1130N/A *trans_r = ctx->ext_trans;

1139N/A return 1;

1139N/A}

1130N/A

1130N/Astatic bool mail_index_sync_view_have_any(struct mail_index_view *view,

1130N/A enum mail_index_sync_flags flags)

970N/A{

970N/A const struct mail_transaction_header *hdr;

970N/A const void *data;

970N/A uint32_t log_seq;

970N/A uoff_t log_offset;

970N/A bool reset;

970N/A int ret;

970N/A

1479N/A if (view->map->hdr.first_recent_uid < view->map->hdr.next_uid &&

1479N/A (flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0)

970N/A return TRUE;

970N/A

970N/A if ((view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) &&

970N/A (flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0)

970N/A return TRUE;

970N/A

970N/A mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset);

970N/A if (mail_transaction_log_view_set(view->log_view,

970N/A view->map->hdr.log_file_seq,

970N/A view->map->hdr.log_file_tail_offset,

970N/A log_seq, log_offset, &reset) <= 0) {

970N/A /* let the actual syncing handle the error */

970N/A return TRUE;

970N/A }

970N/A

1172N/A while ((ret = mail_transaction_log_view_next(view->log_view,

970N/A &hdr, &data)) > 0) {

1443N/A if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)

970N/A continue;

970N/A

970N/A switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {

970N/A case MAIL_TRANSACTION_EXT_REC_UPDATE:

1207N/A case MAIL_TRANSACTION_EXT_ATOMIC_INC:

1207N/A /* extension record updates aren't exactly needed

1207N/A case MAIL_TRANSACTION_EXPUNGE:

1207N/A case MAIL_TRANSACTION_FLAG_UPDATE:

970N/A case MAIL_TRANSACTION_KEYWORD_UPDATE:

970N/A case MAIL_TRANSACTION_KEYWORD_RESET:

1500N/A return TRUE;

1500N/A default:

970N/A break;

970N/A }

970N/A }

1500N/A return ret < 0;

}

bool mail_index_sync_have_any(struct mail_index *index,

enum mail_index_sync_flags flags)

{

struct mail_index_view *view;

bool ret;

(void)mail_index_refresh(index);

view = mail_index_view_open(index);

ret = mail_index_sync_view_have_any(view, flags);

mail_index_view_close(&view);

return ret;

}

void mail_index_sync_get_offsets(struct mail_index_sync_ctx *ctx,

uint32_t *seq1_r, uoff_t *offset1_r,

uint32_t *seq2_r, uoff_t *offset2_r)

{

*seq1_r = ctx->view->map->hdr.log_file_seq;

*offset1_r = ctx->view->map->hdr.log_file_tail_offset != 0 ?

ctx->view->map->hdr.log_file_tail_offset :

ctx->view->index->log->head->hdr.hdr_size;

mail_transaction_log_get_head(ctx->view->index->log, seq2_r, offset2_r);

}

static void

mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,

const struct mail_transaction_expunge *exp)

{

rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;

rec->uid1 = exp->uid1;

rec->uid2 = exp->uid2;

}

static void

mail_index_sync_get_update(struct mail_index_sync_rec *rec,

const struct mail_transaction_flag_update *update)

{

rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;

rec->uid1 = update->uid1;

rec->uid2 = update->uid2;

rec->add_flags = update->add_flags;

rec->remove_flags = update->remove_flags;

}

static void

mail_index_sync_get_keyword_update(struct mail_index_sync_rec *rec,

const struct uid_range *range,

struct mail_index_sync_list *sync_list)

{

rec->type = !sync_list->keyword_remove ?

MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD :

MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;

rec->uid1 = range->uid1;

rec->uid2 = range->uid2;

rec->keyword_idx = sync_list->keyword_idx;

}

static void mail_index_sync_get_keyword_reset(struct mail_index_sync_rec *rec,

const struct uid_range *range)

{

rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;

rec->uid1 = range->uid1;

rec->uid2 = range->uid2;

}

bool mail_index_sync_next(struct mail_index_sync_ctx *ctx,

struct mail_index_sync_rec *sync_rec)

{

struct mail_index_transaction *sync_trans = ctx->sync_trans;

struct mail_index_sync_list *sync_list;

const struct uid_range *uid_range = NULL;

unsigned int i, count, next_i;

uint32_t next_found_uid;

next_i = (unsigned int)-1;

next_found_uid = (uint32_t)-1;

/* FIXME: replace with a priority queue so we don't have to go

sync_list = array_get_modifiable(&ctx->sync_list, &count);

for (i = 0; i < count; i++) {

if (!array_is_created(sync_list[i].array) ||

sync_list[i].idx == array_count(sync_list[i].array))

continue;

uid_range = array_idx(sync_list[i].array, sync_list[i].idx);

if (uid_range->uid1 == ctx->next_uid) {

/* use this one. */

break;

}

if (uid_range->uid1 < next_found_uid) {

next_i = i;

next_found_uid = uid_range->uid1;

}

}

if (i == count) {

if (next_i == (unsigned int)-1) {

/* nothing left in sync_list */

if (ctx->sync_appends) {

ctx->sync_appends = FALSE;

sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;

sync_rec->uid1 = ctx->append_uid_first;

sync_rec->uid2 = ctx->append_uid_last;

return TRUE;

}

return FALSE;

}

ctx->next_uid = next_found_uid;

i = next_i;

uid_range = array_idx(sync_list[i].array, sync_list[i].idx);

}

if (sync_list[i].array == (void *)&sync_trans->expunges) {

mail_index_sync_get_expunge(sync_rec,

(const struct mail_transaction_expunge *)uid_range);

} else if (sync_list[i].array == (void *)&sync_trans->updates) {

mail_index_sync_get_update(sync_rec,

(const struct mail_transaction_flag_update *)uid_range);

} else if (sync_list[i].array == (void *)&sync_trans->keyword_resets) {

mail_index_sync_get_keyword_reset(sync_rec, uid_range);

} else {

mail_index_sync_get_keyword_update(sync_rec, uid_range,

&sync_list[i]);

}

sync_list[i].idx++;

return TRUE;

}

bool mail_index_sync_have_more(struct mail_index_sync_ctx *ctx)

{

const struct mail_index_sync_list *sync_list;

unsigned int i, count;

if (ctx->sync_appends)

return TRUE;

sync_list = array_get(&ctx->sync_list, &count);

for (i = 0; i < count; i++) {

if (array_is_created(sync_list[i].array) &&

sync_list[i].idx != array_count(sync_list[i].array))

return TRUE;

}

return FALSE;

}

void mail_index_sync_reset(struct mail_index_sync_ctx *ctx)

{

struct mail_index_sync_list *sync_list;

unsigned int i, count;

ctx->next_uid = 0;

sync_list = array_get_modifiable(&ctx->sync_list, &count);

for (i = 0; i < count; i++)

sync_list[i].idx = 0;

}

static void mail_index_sync_end(struct mail_index_sync_ctx **_ctx)

{

struct mail_index_sync_ctx *ctx = *_ctx;

i_assert(ctx->index->syncing);

*_ctx = NULL;

ctx->index->syncing = FALSE;

mail_transaction_log_sync_unlock(ctx->index->log);

mail_index_view_close(&ctx->view);

mail_index_transaction_rollback(&ctx->sync_trans);

if (array_is_created(&ctx->sync_list))

array_free(&ctx->sync_list);

i_free(ctx);

}

static void

mail_index_sync_update_mailbox_offset(struct mail_index_sync_ctx *ctx)

{

uint32_t seq;

uoff_t offset;

if (!mail_transaction_log_view_is_last(ctx->view->log_view)) {

/* didn't sync everything */

mail_transaction_log_view_get_prev_pos(ctx->view->log_view,

&seq, &offset);

} else {

/* synced everything, but we might also have committed new

mail_transaction_log_get_head(ctx->index->log, &seq, &offset);

}

mail_transaction_log_set_mailbox_sync_pos(ctx->index->log, seq, offset);

/* If tail offset has changed, make sure it gets written to

if (ctx->last_tail_offset != offset)

ctx->ext_trans->log_updates = TRUE;

}

static bool mail_index_sync_want_index_write(struct mail_index *index)

{

uint32_t log_diff;

log_diff = index->map->hdr.log_file_tail_offset -

index->last_read_log_file_tail_offset;

if (log_diff > MAIL_INDEX_MAX_WRITE_BYTES ||

(index->index_min_write && log_diff > MAIL_INDEX_MIN_WRITE_BYTES))

return TRUE;

if (index->need_recreate)

return TRUE;

return FALSE;

}

int mail_index_sync_commit(struct mail_index_sync_ctx **_ctx)

{

struct mail_index_sync_ctx *ctx = *_ctx;

struct mail_index *index = ctx->index;

uint32_t next_uid;

bool want_rotate;

int ret = 0;

mail_index_sync_update_mailbox_offset(ctx);

if (mail_cache_need_compress(index->cache)) {

/* if cache compression fails, we don't really care.

(void)mail_cache_compress(index->cache, ctx->ext_trans);

}

if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0) {

next_uid = mail_index_transaction_get_next_uid(ctx->ext_trans);

if (index->map->hdr.first_recent_uid < next_uid) {

mail_index_update_header(ctx->ext_trans,

offsetof(struct mail_index_header,

first_recent_uid),

&next_uid, sizeof(next_uid), FALSE);

}

}

if (mail_index_transaction_commit(&ctx->ext_trans) < 0) {

mail_index_sync_end(&ctx);

return -1;

}

/* refresh the mapping with newly committed external transactions

if (mail_index_map(ctx->index, MAIL_INDEX_SYNC_HANDLER_FILE) <= 0)

ret = -1;

want_rotate = mail_transaction_log_want_rotate(index->log);

if (ret == 0 &&

(want_rotate || mail_index_sync_want_index_write(index))) {

index->need_recreate = FALSE;

index->index_min_write = FALSE;

mail_index_write(index, want_rotate);

}

mail_index_sync_end(_ctx);

return ret;

}

void mail_index_sync_rollback(struct mail_index_sync_ctx **ctx)

{

if ((*ctx)->ext_trans != NULL)

mail_index_transaction_rollback(&(*ctx)->ext_trans);

mail_index_sync_end(ctx);

}

void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,

uint8_t *flags)

{

i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);

*flags = (*flags & ~sync_rec->remove_flags) | sync_rec->add_flags;

}

bool mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec,

ARRAY_TYPE(keyword_indexes) *keywords)

{

const unsigned int *keyword_indexes;

unsigned int idx = sync_rec->keyword_idx;

unsigned int i, count;

keyword_indexes = array_get(keywords, &count);

switch (sync_rec->type) {

case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:

for (i = 0; i < count; i++) {

if (keyword_indexes[i] == idx)

return FALSE;

}

array_append(keywords, &idx, 1);

return TRUE;

case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:

for (i = 0; i < count; i++) {

if (keyword_indexes[i] == idx) {

array_delete(keywords, i, 1);

return TRUE;

}

}

return FALSE;

case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:

if (array_count(keywords) == 0)

return FALSE;

array_clear(keywords);

return TRUE;

default:

i_unreached();

return FALSE;

}

}

void mail_index_sync_set_corrupted(struct mail_index_sync_map_ctx *ctx,

const char *fmt, ...)

{

va_list va;

uint32_t seq;

uoff_t offset;

ctx->errors = TRUE;

mail_transaction_log_view_get_prev_pos(ctx->view->log_view,

&seq, &offset);

if (seq < ctx->view->index->fsck_log_head_file_seq ||

(seq == ctx->view->index->fsck_log_head_file_seq &&

offset < ctx->view->index->fsck_log_head_file_offset)) {

/* be silent */

return;

}

va_start(va, fmt);

T_BEGIN {

mail_index_set_error(ctx->view->index,

"Log synchronization error at "

"seq=%u,offset=%"PRIuUOFF_T" for %s: %s",

seq, offset, ctx->view->index->filepath,

t_strdup_vprintf(fmt, va));

} T_END;

va_end(va);

}