mail-index-sync-update.c revision 68b3667c9ee95951d7c3e03b19b2d37abbaa5736
883N/A/* Copyright (c) 2004-2008 Dovecot authors, see the included COPYING file */
883N/A
883N/A#include "lib.h"
883N/A#include "ioloop.h"
883N/A#include "array.h"
883N/A#include "mmap-util.h"
883N/A#include "mail-index-view-private.h"
883N/A#include "mail-index-sync-private.h"
883N/A#include "mail-transaction-log.h"
883N/A#include "mail-transaction-log-private.h"
883N/A
883N/A/* If we have less than this many bytes to sync from log file, don't bother
883N/A reading the main index */
883N/A#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048
883N/A
883N/Astatic void
883N/Amail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
883N/A struct mail_index_map *map, bool eol)
883N/A{
883N/A uint32_t prev_seq;
3996N/A uoff_t prev_offset;
883N/A
883N/A mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
883N/A &prev_seq, &prev_offset);
883N/A
883N/A if (!eol) {
883N/A if (prev_offset == ctx->ext_intro_end_offset &&
883N/A prev_seq == ctx->ext_intro_seq) {
883N/A /* previous transaction was an extension introduction.
883N/A we probably came here from
1273N/A mail_index_sync_ext_reset(). if there are any more
883N/A views which want to continue syncing it needs the
4561N/A intro. so back up a bit more.
4561N/A
4561N/A don't do this in case the last transaction in the
4561N/A log is the extension intro, so we don't keep trying
4561N/A to sync it over and over again. */
4561N/A prev_offset = ctx->ext_intro_offset;
4561N/A }
883N/A } else {
883N/A i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq);
883N/A if (map->hdr.log_file_seq != prev_seq) {
883N/A map->hdr.log_file_seq = prev_seq;
883N/A map->hdr.log_file_tail_offset = 0;
883N/A }
883N/A }
1653N/A map->hdr.log_file_head_offset = prev_offset;
883N/A}
883N/A
883N/Astatic void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
883N/A struct mail_index_map *map)
883N/A{
883N/A struct mail_index_view *view = ctx->view;
883N/A
883N/A i_assert(view->map != map);
883N/A
1574N/A mail_index_sync_update_log_offset(ctx, view->map, FALSE);
1574N/A mail_index_unmap(&view->map);
1574N/A view->map = map;
883N/A
883N/A if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW)
883N/A view->index->map = map;
883N/A}
883N/A
883N/Astatic void
883N/Amail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
883N/A{
883N/A struct mail_index_map *map = ctx->view->map;
883N/A
887N/A i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) ||
887N/A map->rec_map->lock_id != 0);
887N/A
913N/A if (map->refcount > 1) {
913N/A map = mail_index_map_clone(map);
913N/A mail_index_sync_replace_map(ctx, map);
913N/A }
883N/A
3996N/A if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
3996N/A mail_index_map_move_to_memory(ctx->view->map);
3996N/A}
3996N/A
4100N/Astruct mail_index_map *
3996N/Amail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
3996N/A{
3996N/A mail_index_sync_move_to_private_memory(ctx);
3996N/A mail_index_record_map_move_to_private(ctx->view->map);
ctx->view->map->write_atomic = TRUE;
return ctx->view->map;
}
static int
mail_index_header_update_counts(struct mail_index_header *hdr,
uint8_t old_flags, uint8_t new_flags,
const char **error_r)
{
if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
/* different seen-flag */
if ((old_flags & MAIL_SEEN) != 0) {
if (hdr->seen_messages_count == 0) {
*error_r = "Seen counter wrong";
return -1;
}
hdr->seen_messages_count--;
} else {
if (hdr->seen_messages_count >= hdr->messages_count) {
*error_r = "Seen counter wrong";
return -1;
}
if (++hdr->seen_messages_count == hdr->messages_count)
hdr->first_unseen_uid_lowwater = hdr->next_uid;
}
}
if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
/* different deleted-flag */
if ((old_flags & MAIL_DELETED) == 0) {
hdr->deleted_messages_count++;
if (hdr->deleted_messages_count > hdr->messages_count) {
*error_r = "Deleted counter wrong";
return -1;
}
} else {
if (hdr->deleted_messages_count == 0 ||
hdr->deleted_messages_count > hdr->messages_count) {
*error_r = "Deleted counter wrong";
return -1;
}
if (--hdr->deleted_messages_count == 0)
hdr->first_deleted_uid_lowwater = hdr->next_uid;
}
}
return 0;
}
static void
mail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
uint32_t uid,
uint8_t old_flags, uint8_t new_flags)
{
struct mail_index_map *const *maps;
const char *error;
unsigned int i, count;
maps = array_get(&ctx->view->map->rec_map->maps, &count);
for (i = 0; i < count; i++) {
if (uid >= maps[i]->hdr.next_uid)
continue;
if (mail_index_header_update_counts(&maps[i]->hdr,
old_flags, new_flags,
&error) < 0)
mail_index_sync_set_corrupted(ctx, "%s", error);
}
}
static void
mail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
uint32_t uid, uint8_t old_flags,
uint8_t new_flags, bool all)
{
const char *error;
if (all) {
mail_index_sync_header_update_counts_all(ctx, uid, old_flags,
new_flags);
} else {
i_assert(uid < ctx->view->map->hdr.next_uid);
if (mail_index_header_update_counts(&ctx->view->map->hdr,
old_flags, new_flags,
&error) < 0)
mail_index_sync_set_corrupted(ctx, "%s", error);
}
}
static void
mail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
uint32_t uid, enum mail_flags flags)
{
struct mail_index_map *const *maps;
unsigned int i, count;
maps = array_get(&ctx->view->map->rec_map->maps, &count);
for (i = 0; i < count; i++) {
if ((flags & MAIL_SEEN) == 0 &&
uid < maps[i]->hdr.first_unseen_uid_lowwater)
maps[i]->hdr.first_unseen_uid_lowwater = uid;
if ((flags & MAIL_DELETED) != 0 &&
uid < maps[i]->hdr.first_deleted_uid_lowwater)
maps[i]->hdr.first_deleted_uid_lowwater = uid;
}
}
static void
sync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx,
uint32_t seq1, uint32_t seq2)
{
const struct mail_index_expunge_handler *eh;
struct mail_index_record *rec;
unsigned int i, count;
uint32_t seq;
/* call expunge handlers only when syncing index file */
if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE)
return;
if (!ctx->expunge_handlers_set)
mail_index_sync_init_expunge_handlers(ctx);
if (!array_is_created(&ctx->expunge_handlers))
return;
eh = array_get(&ctx->expunge_handlers, &count);
for (i = 0; i < count; i++, eh++) {
for (seq = seq1; seq <= seq2; seq++) {
rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq-1);
/* FIXME: does expunge handler's return value matter?
we probably shouldn't disallow expunges if the
handler returns failure.. should it be just changed
to return void? */
(void)eh->handler(ctx, seq,
PTR_OFFSET(rec, eh->record_offset),
eh->sync_context, eh->context);
}
}
}
static int
sync_expunge(const struct mail_transaction_expunge *e, unsigned int count,
struct mail_index_sync_map_ctx *ctx)
{
struct mail_index_map *map = ctx->view->map;
struct mail_index_record *rec;
uint32_t seq_count, seq, seq1, seq2;
unsigned int i;
for (i = 0; i < count; i++, e++) {
if (!mail_index_lookup_seq_range(ctx->view, e->uid1, e->uid2,
&seq1, &seq2)) {
/* everything expunged already */
continue;
}
sync_expunge_call_handlers(ctx, seq1, seq2);
map = mail_index_sync_get_atomic_map(ctx);
for (seq = seq1; seq <= seq2; seq++) {
rec = MAIL_INDEX_MAP_IDX(map, seq-1);
mail_index_sync_header_update_counts(ctx, rec->uid,
rec->flags, 0,
FALSE);
}
/* @UNSAFE */
memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
MAIL_INDEX_MAP_IDX(map, seq2),
(map->rec_map->records_count - seq2) *
map->hdr.record_size);
seq_count = seq2 - seq1 + 1;
map->rec_map->records_count -= seq_count;
map->hdr.messages_count -= seq_count;
}
return 1;
}
void mail_index_sync_write_seq_update(struct mail_index_sync_map_ctx *ctx,
uint32_t seq1, uint32_t seq2)
{
struct mail_index_map *map = ctx->view->map;
if (map->rec_map->write_seq_first == 0 ||
map->rec_map->write_seq_first > seq1)
map->rec_map->write_seq_first = seq1;
if (map->rec_map->write_seq_last < seq2)
map->rec_map->write_seq_last = seq2;
}
static int sync_append(const struct mail_index_record *rec,
struct mail_index_sync_map_ctx *ctx)
{
struct mail_index_view *view = ctx->view;
struct mail_index_map *map = view->map;
enum mail_flags new_flags;
void *dest;
size_t append_pos;
if (rec->uid < map->hdr.next_uid) {
mail_index_sync_set_corrupted(ctx,
"Append with UID %u, but next_uid = %u",
rec->uid, map->hdr.next_uid);
return -1;
}
/* move to memory. the mapping is written when unlocking so we don't
waste time re-mmap()ing multiple times or waste space growing index
file too large */
mail_index_sync_move_to_private_memory(ctx);
map = view->map;
if (rec->uid <= map->rec_map->last_appended_uid) {
i_assert(map->hdr.messages_count < map->rec_map->records_count);
/* the flags may have changed since it was added to map.
use the updated flags already, so flag counters won't get
broken. */
new_flags =
MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count)->flags;
} else {
/* don't rely on buffer->used being at the correct position.
at least expunges can move it */
append_pos = map->rec_map->records_count * map->hdr.record_size;
dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
map->hdr.record_size);
map->rec_map->records =
buffer_get_modifiable_data(map->rec_map->buffer, NULL);
memcpy(dest, rec, sizeof(*rec));
memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
map->hdr.record_size - sizeof(*rec));
map->rec_map->records_count++;
map->rec_map->last_appended_uid = rec->uid;
new_flags = rec->flags;
mail_index_sync_write_seq_update(ctx,
map->rec_map->records_count,
map->rec_map->records_count);
}
map->hdr.messages_count++;
map->hdr.next_uid = rec->uid+1;
if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
mail_index_header_update_lowwaters(ctx, rec->uid, new_flags);
mail_index_sync_header_update_counts(ctx, rec->uid,
0, new_flags, FALSE);
return 1;
}
static int sync_flag_update(const struct mail_transaction_flag_update *u,
struct mail_index_sync_map_ctx *ctx)
{
struct mail_index_view *view = ctx->view;
struct mail_index_record *rec;
uint8_t flag_mask, old_flags;
uint32_t idx, seq1, seq2;
if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2))
return 1;
mail_index_sync_write_seq_update(ctx, seq1, seq2);
if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
flag_mask = ~u->remove_flags;
if (((u->add_flags | u->remove_flags) &
(MAIL_SEEN | MAIL_DELETED)) == 0) {
/* we're not modifying any counted/lowwatered flags */
for (idx = seq1-1; idx < seq2; idx++) {
rec = MAIL_INDEX_MAP_IDX(view->map, idx);
rec->flags = (rec->flags & flag_mask) | u->add_flags;
}
} else {
for (idx = seq1-1; idx < seq2; idx++) {
rec = MAIL_INDEX_MAP_IDX(view->map, idx);
old_flags = rec->flags;
rec->flags = (rec->flags & flag_mask) | u->add_flags;
mail_index_header_update_lowwaters(ctx, rec->uid,
rec->flags);
mail_index_sync_header_update_counts(ctx, rec->uid,
old_flags,
rec->flags, TRUE);
}
}
return 1;
}
static int sync_header_update(const struct mail_transaction_header_update *u,
struct mail_index_sync_map_ctx *ctx)
{
#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \
((u)->offset <= offsetof(struct mail_index_header, field) && \
(u)->offset + (u)->size > offsetof(struct mail_index_header, field))
struct mail_index_map *map = ctx->view->map;
uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset;
uint32_t orig_next_uid = map->hdr.next_uid;
uint32_t orig_uid_validity = map->hdr.uid_validity;
if (u->offset >= map->hdr.base_header_size ||
u->offset + u->size > map->hdr.base_header_size) {
mail_index_sync_set_corrupted(ctx,
"Header update outside range: %u + %u > %u",
u->offset, u->size, map->hdr.base_header_size);
return -1;
}
buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
map->hdr_base = map->hdr_copy_buf->data;
map->write_base_header = TRUE;
/* @UNSAFE */
if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {
memcpy(PTR_OFFSET(&map->hdr, u->offset),
u + 1, u->size);
} else if (u->offset < sizeof(map->hdr)) {
memcpy(PTR_OFFSET(&map->hdr, u->offset),
u + 1, sizeof(map->hdr) - u->offset);
}
/* UIDVALIDITY can be changed only by resetting the index */
if (orig_uid_validity != 0 &&
MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, uid_validity)) {
mail_index_sync_set_corrupted(ctx,
"uid_validity updated unexpectedly: %u -> %u",
orig_uid_validity, map->hdr.uid_validity);
/* let it through anyway, although this could give wrong
"next_uid shrank" errors if the value actually changed.. */
}
if (map->hdr.next_uid < orig_next_uid) {
mail_index_sync_set_corrupted(ctx,
"next_uid shrank ignored: %u -> %u",
orig_next_uid, map->hdr.next_uid);
map->hdr.next_uid = orig_next_uid;
return -1;
}
/* the tail offset updates are intended for internal transaction
log handling. we'll update the offset in the header only when
the sync is finished. */
map->hdr.log_file_tail_offset = orig_log_file_tail_offset;
return 1;
}
int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_header *hdr,
const void *data)
{
int ret = 0;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
const struct mail_index_record *rec, *end;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec < end; rec++) {
ret = sync_append(rec, ctx);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_EXPUNGE:
case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
const struct mail_transaction_expunge *rec = data, *end;
if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
/* this is simply a request for expunge */
break;
}
end = CONST_PTR_OFFSET(data, hdr->size);
ret = sync_expunge(rec, end - rec, ctx);
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *rec, *end;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec < end; rec++) {
ret = sync_flag_update(rec, ctx);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_HEADER_UPDATE: {
const struct mail_transaction_header_update *rec;
unsigned int i;
for (i = 0; i < hdr->size; ) {
rec = CONST_PTR_OFFSET(data, i);
ret = sync_header_update(rec, ctx);
if (ret <= 0)
break;
i += sizeof(*rec) + rec->size;
if ((i % 4) != 0)
i += 4 - (i % 4);
}
break;
}
case MAIL_TRANSACTION_EXT_INTRO: {
const struct mail_transaction_ext_intro *rec = data;
unsigned int i;
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
&prev_seq, &prev_offset);
ctx->ext_intro_seq = prev_seq;
ctx->ext_intro_offset = prev_offset;
ctx->ext_intro_end_offset =
prev_offset + hdr->size + sizeof(*hdr);
for (i = 0; i < hdr->size; ) {
if (i + sizeof(*rec) > hdr->size) {
/* should be just extra padding */
break;
}
rec = CONST_PTR_OFFSET(data, i);
if (i + sizeof(*rec) + rec->name_size > hdr->size) {
mail_index_sync_set_corrupted(ctx,
"ext intro: name_size too large");
ret = -1;
break;
}
ret = mail_index_sync_ext_intro(ctx, rec);
if (ret <= 0)
break;
i += sizeof(*rec) + rec->name_size;
if ((i % 4) != 0)
i += 4 - (i % 4);
}
break;
}
case MAIL_TRANSACTION_EXT_RESET: {
const struct mail_transaction_ext_reset *rec = data;
if (hdr->size != sizeof(*rec)) {
mail_index_sync_set_corrupted(ctx,
"ext reset: invalid record size");
ret = -1;
break;
}
ret = mail_index_sync_ext_reset(ctx, rec);
break;
}
case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
const struct mail_transaction_ext_hdr_update *rec = data;
unsigned int i;
for (i = 0; i < hdr->size; ) {
rec = CONST_PTR_OFFSET(data, i);
if (i + sizeof(*rec) > hdr->size ||
i + sizeof(*rec) + rec->size > hdr->size) {
mail_index_sync_set_corrupted(ctx,
"ext hdr update: invalid record size");
ret = -1;
break;
}
ret = mail_index_sync_ext_hdr_update(ctx, rec);
if (ret <= 0)
break;
i += sizeof(*rec) + rec->size;
if ((i % 4) != 0)
i += 4 - (i % 4);
}
break;
}
case MAIL_TRANSACTION_EXT_REC_UPDATE: {
const struct mail_transaction_ext_rec_update *rec;
const struct mail_index_ext *ext;
unsigned int i, record_size;
if (ctx->cur_ext_map_idx == (uint32_t)-1) {
mail_index_sync_set_corrupted(ctx,
"Extension record updated "
"without intro prefix");
ret = -1;
break;
}
if (ctx->cur_ext_ignore) {
ret = 1;
break;
}
ext = array_idx(&ctx->view->map->extensions,
ctx->cur_ext_map_idx);
/* the record is padded to 32bits in the transaction log */
record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
for (i = 0; i < hdr->size; i += record_size) {
rec = CONST_PTR_OFFSET(data, i);
if (i + record_size > hdr->size) {
mail_index_sync_set_corrupted(ctx,
"ext rec update: invalid record size");
ret = -1;
break;
}
ret = mail_index_sync_ext_rec_update(ctx, rec);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_KEYWORD_UPDATE: {
const struct mail_transaction_keyword_update *rec = data;
ret = mail_index_sync_keywords(ctx, hdr, rec);
break;
}
case MAIL_TRANSACTION_KEYWORD_RESET: {
const struct mail_transaction_keyword_reset *rec = data;
ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
break;
}
default:
mail_index_sync_set_corrupted(ctx,
"Unknown transaction record type 0x%x",
(hdr->type & MAIL_TRANSACTION_TYPE_MASK));
ret = -1;
break;
}
return ret;
}
void mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx,
struct mail_index_view *view,
enum mail_index_sync_handler_type type)
{
memset(sync_map_ctx, 0, sizeof(*sync_map_ctx));
sync_map_ctx->view = view;
sync_map_ctx->cur_ext_map_idx = (uint32_t)-1;
sync_map_ctx->type = type;
mail_index_sync_init_handlers(sync_map_ctx);
}
void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx)
{
if (sync_map_ctx->unknown_extensions != NULL)
buffer_free(&sync_map_ctx->unknown_extensions);
if (sync_map_ctx->expunge_handlers_used)
mail_index_sync_deinit_expunge_handlers(sync_map_ctx);
mail_index_sync_deinit_handlers(sync_map_ctx);
}
static void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map)
{
const struct mail_index_record *rec;
unsigned int i;
if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0)
return;
/* do we have dirty flags anymore? */
for (i = 0; i < map->rec_map->records_count; i++) {
rec = MAIL_INDEX_MAP_IDX(map, i);
if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
break;
}
}
}
#ifdef DEBUG
void mail_index_map_check(struct mail_index_map *map)
{
const struct mail_index_header *hdr = &map->hdr;
unsigned int i, del = 0, seen = 0;
i_assert(hdr->messages_count <= map->rec_map->records_count);
for (i = 0; i < hdr->messages_count; i++) {
const struct mail_index_record *rec;
rec = MAIL_INDEX_MAP_IDX(map, i);
if (rec->flags & MAIL_DELETED) {
i_assert(rec->uid >= hdr->first_deleted_uid_lowwater);
del++;
}
if (rec->flags & MAIL_SEEN)
seen++;
else
i_assert(rec->uid >= hdr->first_unseen_uid_lowwater);
}
i_assert(del == hdr->deleted_messages_count);
i_assert(seen == hdr->seen_messages_count);
}
#endif
int mail_index_sync_map(struct mail_index_map **_map,
enum mail_index_sync_handler_type type, bool force)
{
struct mail_index_map *map = *_map;
struct mail_index *index = map->index;
struct mail_index_view *view;
struct mail_index_sync_map_ctx sync_map_ctx;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq;
uoff_t start_offset, prev_offset;
int ret;
bool had_dirty, reset;
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (index->log->head == NULL) {
i_assert(!force);
return 0;
}
start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
if (!force && !index->mmap_disable) {
/* see if we'd prefer to reopen the index file instead of
syncing the current map from the transaction log.
don't check this if mmap is disabled, because reopening
index causes sync to get lost. */
uoff_t log_size, index_size;
if (index->fd == -1 &&
index->log->head->hdr.prev_file_seq != 0) {
/* we don't know the index's size, so use the
smallest index size we're willing to read */
index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
} else {
index_size = map->hdr.header_size +
map->rec_map->records_count *
map->hdr.record_size;
}
/* this isn't necessary correct currently, but it should be
close enough */
log_size = index->log->head->last_size;
if (log_size > start_offset &&
log_size - start_offset > index_size)
return 0;
}
view = mail_index_view_open_with_map(index, map);
ret = mail_transaction_log_view_set(view->log_view,
map->hdr.log_file_seq, start_offset,
(uint32_t)-1, (uoff_t)-1, &reset);
if (ret <= 0) {
if (force && ret == 0) {
/* the seq/offset is probably broken */
(void)mail_index_fsck(index);
}
/* can't use it. sync by re-reading index. */
mail_index_view_close(&view);
return 0;
}
/* view referenced the map. avoid unnecessary map cloning by
unreferencing the map while view exists. */
map->refcount--;
had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty) {
map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
map->write_base_header = TRUE;
}
if (map->hdr_base != map->hdr_copy_buf->data) {
/* if syncing updates the header, it updates hdr_copy_buf
and updates hdr_base to hdr_copy_buf. so the buffer must
initially contain a valid header or we'll break it when
writing it. */
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, map->hdr_base,
map->hdr.header_size);
map->hdr_base = map->hdr_copy_buf->data;
}
mail_index_sync_map_init(&sync_map_ctx, view, type);
if (reset) {
/* Reset the entire index. Leave only indexid and
log_file_seq. */
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
map = mail_index_map_alloc(index);
map->hdr.log_file_seq = prev_seq;
map->hdr.log_file_tail_offset = 0;
mail_index_sync_replace_map(&sync_map_ctx, map);
}
map = NULL;
/* FIXME: when transaction sync lock is removed, we'll need to handle
the case when a transaction is committed while mailbox is being
synced ([synced transactions][new transaction][ext transaction]).
this means int_offset contains [synced] and ext_offset contains
all */
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&tdata)) > 0) {
mail_transaction_log_view_get_prev_pos(view->log_view,
&prev_seq, &prev_offset);
if (LOG_IS_BEFORE(prev_seq, prev_offset,
view->map->hdr.log_file_seq,
view->map->hdr.log_file_head_offset)) {
/* this has been synced already. we're here only to call
expunge handlers and extension update handlers. */
i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
continue;
if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
continue;
}
/* we'll just skip over broken entries */
T_BEGIN {
(void)mail_index_sync_record(&sync_map_ctx,
thdr, tdata);
} T_END;
}
map = view->map;
if (had_dirty)
mail_index_sync_update_hdr_dirty_flag(map);
mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE);
#ifdef DEBUG
mail_index_map_check(map);
#endif
i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0);
/* transaction log tracks internally the current tail offset.
besides using header updates, it also updates the offset to skip
over following external transactions to avoid extra unneeded log
reading. */
i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq);
if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) {
map->hdr.log_file_tail_offset =
index->log->head->max_tail_offset;
}
buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
map->hdr_copy_buf->used);
}
/* restore refcount before closing the view. this is necessary also
if map got cloned, because view closing would otherwise destroy it */
map->refcount++;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW);
if (mail_index_map_check_header(map) <= 0) {
mail_index_set_error(index,
"Synchronization corrupted index header: %s",
index->filepath);
(void)mail_index_fsck(index);
map = index->map;
} else if (sync_map_ctx.errors) {
/* make sure the index looks valid now */
(void)mail_index_fsck(index);
map = index->map;
}
*_map = map;
return ret < 0 ? -1 : 1;
}