mail-index-view-sync.c revision 6f4ddcbd1961beb71a7e2b3e12f5af26942671c0
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek/* Copyright (C) 2003-2004 Timo Sirainen */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek const void *data;
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozekmail_transaction_log_sort_expunges(ARRAY_TYPE(seq_range) *expunges,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* Note that all the sequences are actually still UIDs at this point */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina /* @UNSAFE */
80941dd89fd8bc7c4a1272c304f737ce0fd5fc54Sumit Bose dest = array_get_modifiable(expunges, &dest_count);
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina array_append(expunges, src, src_size / sizeof(*src));
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina /* src[] must be sorted. */
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina (src+1 != src_end && src->seq2 >= src[1].seq1))
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina for (; i < dest_count; i++) {
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina while (i < dest_count && src->seq2 >= dest[i].seq1-1) {
80941dd89fd8bc7c4a1272c304f737ce0fd5fc54Sumit Bose /* we can/must merge with next record */
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina if (first > 0 && new_exp.seq1 <= dest[first-1].seq2+1) {
5a2cce34cf8843613b0b9dfde054b3d471dd5f3aPavel Březina /* continue previous record */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina } else if (i == first) {
db419c61035cb262010cc8d5a4047191c2b60f05Pavel Březina dest = array_get_modifiable(expunges, &dest_count);
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek /* use next record */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina dest = array_get_modifiable(expunges, &dest_count);
db419c61035cb262010cc8d5a4047191c2b60f05Pavel Březinaview_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges)
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina const struct mail_index_header *hdr = &view->index->map->hdr;
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina /* the view begins from the first non-synced transaction */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina ret = mail_transaction_log_view_set(view->log_view,
ed44814e0e7ff9f0ef7ffc98fab7d9542a7822dfPavel Březina /* FIXME: use the new index to get needed
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek "Transaction log got desynced for index %s",
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek /* we can't do this. sync only up to reset. */
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek mail_transaction_log_view_get_prev_pos(view->log_view,
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina /* we have only this reset log */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina mail_transaction_log_view_clear(view->log_view);
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozekview_sync_get_expunges(struct mail_index_view *view,
db419c61035cb262010cc8d5a4047191c2b60f05Pavel Březina if (view_sync_set_log_view_range(view, TRUE) < 0)
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina /* get a list of expunge transactions. there may be some that we have
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina already synced, but it doesn't matter because they'll get dropped
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek out when converting to sequences */
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek while ((ret = mail_transaction_log_view_next(view->log_view,
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina /* this is simply a request for expunge */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina if (mail_transaction_log_sort_expunges(expunges_r, data,
4be402505ba20b43361753f0e6e1589c9b029e81Jakub Hrozek mail_transaction_log_view_set_corrupted(view->log_view,
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina "Corrupted expunge record");
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina /* convert UIDs to sequences */
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina src = dest = array_get_modifiable(expunges_r, &count);
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina ret = mail_index_lookup_uid_range(view, src->seq1,
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina array_delete(expunges_r, count, array_count(expunges_r) - count);
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březinastatic int have_existing_expunges(struct mail_index_view *view,
c9aab1c04c399ca2d1abef74f6df22ced34983dcPavel Březina if (mail_index_lookup_uid_range(view, range->seq1, range->seq2,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozekstatic bool view_sync_have_expunges(struct mail_index_view *view)
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek mail_transaction_log_view_get_prev_pos(view->log_view,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek while ((ret = mail_transaction_log_view_next(view->log_view,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0)
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina /* this is simply a request for expunge */
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina /* we have an expunge. see if it still exists. */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek ret = have_existing_expunges(view, data, hdr->size);
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina mail_transaction_log_view_seek(view->log_view, seq, offset);
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* handle failures as having expunges (which is safer).
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina we'll probably fail later. */
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina return ret != 0;
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březinaint mail_index_view_sync_begin(struct mail_index_view *view,
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina unsigned int expunge_count = 0;
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina if (mail_index_map_lock(view->index->map) < 0)
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* get list of all expunges first */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek if (view_sync_set_log_view_range(view, sync_expunges) < 0) {
f6171b2bc954a367f316853ab71090eb213bdee3Pavel Březina ctx = i_new(struct mail_index_view_sync_ctx, 1);
f7af8c5b369938725e47585c641ae5b017d442a1Pavel Březina mail_index_sync_map_init(&ctx->sync_map_ctx, view,
f7af8c5b369938725e47585c641ae5b017d442a1Pavel Březina if (sync_expunges || !view_sync_have_expunges(view)) {
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* keep the old mapping without expunges until we're
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek fully synced */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* We need a private copy of the map if we don't want to
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek sync expunges.
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek If view's map is the head map, it means that it contains
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek already all the latest changes and there's no need for us
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek to apply any changes to it. This can only happen if there
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek hadn't been any expunges. */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* Using non-head mapping. We have to apply
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek transactions to it to get latest changes into it. */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek i_assert(map->records_count == map->hdr.messages_count);
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* Syncing the view invalidates all previous looked up records.
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek Unreference the mappings this view keeps because of them. */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozekview_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina const struct mail_index_view_log_sync_area *syncs;
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina unsigned int i, count;
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek syncs = array_get(&view->syncs_hidden, &count);
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek for (i = 0; i < count; i++) {
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina offset - syncs[i].log_file_offset < syncs[i].length &&
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březinamail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina next_offset = offset + sizeof(*hdr) + hdr->size;
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
e7b5b99e5a5d276f32039c4fb8b21ba51bdb1537Pavel Březina if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
e7b5b99e5a5d276f32039c4fb8b21ba51bdb1537Pavel Březina /* already synced */
e7b5b99e5a5d276f32039c4fb8b21ba51bdb1537Pavel Březina if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
e7b5b99e5a5d276f32039c4fb8b21ba51bdb1537Pavel Březina /* already synced */
e7b5b99e5a5d276f32039c4fb8b21ba51bdb1537Pavel Březinamail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
e7b5b99e5a5d276f32039c4fb8b21ba51bdb1537Pavel Březina struct mail_transaction_log_view *log_view = ctx->view->log_view;
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina /* Get the next transaction from log. */
cda8ff6cfdef22356dc3c06ec5204344912f0f0bPavel Březina ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* This is a visible record that we don't want to
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* If we started from a map that we didn't create ourself,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek some of the transactions may already be synced. at the end
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek of this view sync we'll update file_seq=0 so that this check
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek always becomes FALSE for subsequent syncs. */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek synced_to_map = view->map->hdr.log_file_seq != 0 &&
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* Apply transaction to view's mapping if needed (meaning we
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek didn't just re-map the view to head mapping). */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek i_assert((hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0 ||
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* skip changes committed by hidden transactions (eg. in IMAP
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina store +flags.silent command) */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březinamail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina const struct mail_transaction_header *hdr = ctx->hdr;
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* data contains the appended records, but we don't care */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* this is simply a request for expunge */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* data contains mail_transaction_expunge[] */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina const struct mail_transaction_flag_update *update =
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* data contains mail_transaction_flag_update[] */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* skip internal flag changes */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina update = CONST_PTR_OFFSET(data, ctx->data_offset);
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina const struct mail_transaction_keyword_update *update = data;
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* data contains mail_transaction_keyword_update header,
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina the keyword name and an array of { uint32_t uid1, uid2; } */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* skip over the header and name */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina ctx->data_offset = sizeof(*update) + update->name_size;
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina ctx->data_offset += 4 - (ctx->data_offset % 4);
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina uids = CONST_PTR_OFFSET(data, ctx->data_offset);
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina const struct mail_transaction_keyword_reset *reset =
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* data contains mail_transaction_keyword_reset[] */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březinaint mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina ret = mail_index_view_sync_get_next_transaction(ctx);
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina } while (!mail_index_view_sync_get_rec(ctx, sync_rec));
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březinavoid mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březinamail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina const struct mail_index_view_log_sync_area *syncs;
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina unsigned int i, count;
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina /* Clean up to view's tail */
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina syncs = array_get(&view->syncs_hidden, &count);
44749ce0c1fee9babee80060fa0db99eebb2ab51Pavel Březina for (i = 0; i < count; i++) {
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březina syncs[i].length > view->log_file_expunge_offset &&
9675bccabff4e79d224f64611ad9ff3e073b488eSimo Sorce syncs[i].log_file_seq == view->log_file_expunge_seq) ||
9675bccabff4e79d224f64611ad9ff3e073b488eSimo Sorce syncs[i].log_file_seq > view->log_file_expunge_seq)
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březinavoid mail_index_view_sync_end(struct mail_index_view_sync_ctx **_ctx)
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březina /* we didn't sync everything */
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březina i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březina view->log_file_expunge_seq = view->log_file_head_seq;
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březina view->log_file_expunge_offset = view->log_file_head_offset;
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* log offsets have no meaning in views. make sure they're not
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek tried to be used wrong by setting them to zero. */
f5d4b05027acce06e3509ecb68869d1c7ef37180Pavel Březina mail_index_sync_map_deinit(&ctx->sync_map_ctx);
f643754db81eeade60485bbe3d80324d889cc4f3Pavel Březina mail_index_view_sync_clean_log_syncs(ctx->view);
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek /* set log view to empty range so unneeded memory gets freed */
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozekvoid mail_index_view_add_hidden_transaction(struct mail_index_view *view,
3f98cdc011bb4e8cd22c088f288b0bcdb6452492Jakub Hrozek unsigned int length)