src/lib-index/mail-index.c

	mail-index.c revision b62397096a70a9103dbcffd75b7be50501dbce04
a8c5a86d183db25a57bf193c06b41e092ec2e151Timo Sirainen/* Copyright (C) 2002 Timo Sirainen */
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "lib.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "ioloop.h"
0536ccb51d41e3078c3a9fa33e509fb4b2420f95Timo Sirainen#include "file-lock.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "file-set-size.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mmap-util.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-index.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-index-data.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-index-util.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-tree.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-modifylog.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-custom-flags.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen#include <unistd.h>
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen#include <fcntl.h>
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen#include <utime.h>
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenstatic int mmap_verify(MailIndex *index)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen    MailIndexHeader *hdr;
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen    unsigned int extra;
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen    index->mmap_used_length = 0;
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen    if (index->mmap_full_length < sizeof(MailIndexHeader)) {
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen                index_set_corrupted(index, "File too small");
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        return FALSE;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    }
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    extra = (index->mmap_full_length - sizeof(MailIndexHeader)) %
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        sizeof(MailIndexRecord);
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    if (extra != 0) {
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        /* partial write or corrupted -
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen           truncate the file to valid length */
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        i_assert(!index->anon_mmap);
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        index->mmap_full_length -= extra;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        (void)ftruncate(index->fd, (off_t)index->mmap_full_length);
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    }
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    /* keep the header set even if we fail, so we can update the flags */
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    hdr = index->mmap_base;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    index->header = hdr;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen    if (hdr->used_file_size > index->mmap_full_length) {
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen        index_set_corrupted(index, "used_file_size larger than real "
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen                    "file size (%"PRIuUOFF_T" vs %"PRIuSIZE_T
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen                    ")", hdr->used_file_size,
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen                    index->mmap_full_length);
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen        return FALSE;
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen    }
59e26ff34b05cd971a111f8a42fc60c13d9f688bTimo Sirainen
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen    if ((hdr->used_file_size - sizeof(MailIndexHeader)) %
2d2ebe91d56e9a158de000c9d0026f65600fbcfaTimo Sirainen        sizeof(MailIndexRecord) != 0) {
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen        index_set_corrupted(index, "Invalid used_file_size in header "
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen                    "(%"PRIuUOFF_T")",
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen                    hdr->used_file_size);
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen        return FALSE;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen    }
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen    if (hdr->messages_count < hdr->seen_messages_count) {
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen        index_set_corrupted(index, "Invalid seen messages count "
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen                    "(%u < %u)", hdr->messages_count,
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen                    hdr->seen_messages_count);
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen        return FALSE;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen    }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen    if (hdr->messages_count < hdr->deleted_messages_count) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index_set_corrupted(index, "Invalid deleted messages count "
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen                    "(%u < %u)", hdr->messages_count,
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen                    hdr->deleted_messages_count);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        return FALSE;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
571fd6ff94570ee11a72a20b649acfdac2495919Timo Sirainen    index->sync_id = hdr->sync_id;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    index->mmap_used_length = hdr->used_file_size;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    return TRUE;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen}
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainenint mail_index_mmap_update(MailIndex *index)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen{
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    if (index->anon_mmap)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        return mmap_verify(index);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (index->mmap_base != NULL) {
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen        index->header = (MailIndexHeader *) index->mmap_base;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen        /* make sure file size hasn't changed */
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        if (index->header->sync_id == index->sync_id) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen            index->mmap_used_length = index->header->used_file_size;
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen            if (index->mmap_used_length > index->mmap_full_length) {
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen                i_panic("Index file size was grown without "
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen                    "updating sync_id");
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen            }
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen            return TRUE;
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen        }
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        if (msync(index->mmap_base,
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen              index->mmap_used_length, MS_SYNC) < 0)
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen            return index_set_syscall_error(index, "msync()");
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen        if (munmap(index->mmap_base, index->mmap_full_length) < 0)
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen            return index_set_syscall_error(index, "munmap()");
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen    }
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen    index->mmap_base = mmap_rw_file(index->fd, &index->mmap_full_length);
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen    if (index->mmap_base == MAP_FAILED) {
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen        index->mmap_base = NULL;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->mmap_used_length = 0;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index_set_syscall_error(index, "mmap()");
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        return FALSE;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    }
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen    return mmap_verify(index);
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen}
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainenvoid mail_index_close(MailIndex *index)
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen{
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    index->set_flags = 0;
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen    index->set_cache_fields = 0;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen    index->opened = FALSE;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen    index->inconsistent = FALSE;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    index->lock_type = MAIL_LOCK_UNLOCK;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    index->header = NULL;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen    if (index->fd != -1) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        if (close(index->fd) < 0)
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen            index_set_syscall_error(index, "close()");
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->fd = -1;
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen    }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (index->filepath != NULL) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        i_free(index->filepath);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->filepath = NULL;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    }
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    if (index->anon_mmap) {
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        if (munmap_anon(index->mmap_base, index->mmap_full_length) < 0)
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen            index_set_syscall_error(index, "munmap_anon()");
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        index->anon_mmap = FALSE;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    } else if (index->mmap_base != NULL) {
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen        if (munmap(index->mmap_base, index->mmap_full_length) < 0)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen            index_set_syscall_error(index, "munmap()");
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen    }
8f32e59200da904613506f5649ffa4d9f5989cebTimo Sirainen    index->mmap_base = NULL;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    if (index->data != NULL) {
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen                mail_index_data_free(index->data);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->data = NULL;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    if (index->tree != NULL) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                mail_tree_free(index->tree);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index->tree = NULL;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen    if (index->modifylog != NULL) {
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen                mail_modifylog_free(index->modifylog);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->modifylog = NULL;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    }
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen    if (index->custom_flags != NULL) {
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen        mail_custom_flags_free(index->custom_flags);
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen                index->custom_flags = NULL;
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen    }
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen    if (index->error != NULL) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        i_free(index->error);
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen        index->error = NULL;
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen    }
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainenstatic int mail_index_sync_file(MailIndex *index)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    unsigned int i;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    int failed, fsync_fds[3];
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen    if (index->anon_mmap)
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen        return TRUE;
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen    for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++)
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen        fsync_fds[i] = -1;
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen    if (!mail_index_data_sync_file(index->data, &fsync_fds[0]))
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen        return FALSE;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (msync(index->mmap_base, index->mmap_used_length, MS_SYNC) < 0)
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen        return index_set_syscall_error(index, "msync()");
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    failed = FALSE;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (index->tree != NULL) {
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen        if (!mail_tree_sync_file(index->tree, &fsync_fds[1]))
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen            failed = TRUE;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (index->modifylog != NULL) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        if (!mail_modifylog_sync_file(index->modifylog, &fsync_fds[2]))
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen            failed = TRUE;
8f32e59200da904613506f5649ffa4d9f5989cebTimo Sirainen    }
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen    for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        if (fsync_fds[i] != -1 && fdatasync(fsync_fds[i]) < 0)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen            index_set_error(index, "fdatasync(%u) failed: %m", i);
b92e979748a22925b0770d3004eaab043ed69574Timo Sirainen    }
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen    if (fdatasync(index->fd) < 0)
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen        return index_set_syscall_error(index, "fdatasync()");
35283613d4c04ce18836e9fc431582c87b3710a0Timo Sirainen
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen    return !failed;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen}
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
35283613d4c04ce18836e9fc431582c87b3710a0Timo Sirainenstatic void mail_index_update_timestamp(MailIndex *index)
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen{
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen    struct utimbuf ut;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen    /* keep index's modify stamp same as the sync file's stamp */
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    ut.actime = ioloop_time;
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen    ut.modtime = index->file_sync_stamp;
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen    if (utime(index->filepath, &ut) < 0)
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen        index_set_syscall_error(index, "utime()");
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainenint mail_index_fmdatasync(MailIndex *index, size_t size)
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen{
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (!index->anon_mmap) {
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        if (msync(index->mmap_base, size, MS_SYNC) < 0)
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen            return index_set_syscall_error(index, "msync()");
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen        if (fdatasync(index->fd) < 0)
0dbcf4026ff8471b4d6d2e14f7e52321396bf087Timo Sirainen            return index_set_syscall_error(index, "fdatasync()");
0dbcf4026ff8471b4d6d2e14f7e52321396bf087Timo Sirainen    }
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen    return TRUE;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen}
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainenstatic void mail_index_update_header_changes(MailIndex *index)
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen{
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    if (index->set_flags != 0) {
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen        index->header->flags |= index->set_flags;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen        index->set_flags = 0;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen    }
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen    if (index->set_cache_fields != 0) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->header->cache_fields = index->set_cache_fields;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        index->set_cache_fields = 0;
b92e979748a22925b0770d3004eaab043ed69574Timo Sirainen    }
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen}
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainenstatic int mail_index_write_header_changes(MailIndex *index)
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen{
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    int failed;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    /* use our own locking here so we don't mess up with any other
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen       index states, like inconsistency. */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    if (file_wait_lock(index->fd, F_WRLCK, DEFAULT_LOCK_TIMEOUT) <= 0)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        return index_set_syscall_error(index, "file_wait_lock()");
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen#ifdef DEBUG
4e8d6d03c2ff85448df79b181a2ea850fb5d4199Timo Sirainen    mprotect(index->mmap_base, index->mmap_used_length,
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen         PROT_READ|PROT_WRITE);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen#endif
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    mail_index_update_header_changes(index);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    failed = msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    if (failed)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        index_set_syscall_error(index, "msync()");
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen#ifdef DEBUG
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    mprotect(index->mmap_base, index->mmap_used_length, PROT_NONE);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen#endif
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    if (file_wait_lock(index->fd, F_UNLCK, 0) <= 0)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        return index_set_syscall_error(index, "file_wait_lock()");
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    return !failed;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen}
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainenstatic int mail_index_lock_remove(MailIndex *index)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen{
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    MailLockType old_lock_type;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    if (file_wait_lock(index->fd, F_UNLCK, 0) <= 0)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        return index_set_syscall_error(index, "file_wait_lock()");
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    old_lock_type = index->lock_type;
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen    index->lock_type = MAIL_LOCK_UNLOCK;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    if (old_lock_type == MAIL_LOCK_SHARED) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        /* releasing shared lock. we may need to update some
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen           flags in header. */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        unsigned int old_flags, old_cache;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen        old_flags = index->header->flags;
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen        old_cache = index->header->cache_fields;
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen        if ((old_flags | index->set_flags) != old_flags ||
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen            (old_cache | index->set_cache_fields) != old_cache)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen            return mail_index_write_header_changes(index);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        debug_mprotect(index->mmap_base, index->mmap_full_length, index);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen    return TRUE;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen}
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainenstatic int mail_index_lock_change(MailIndex *index, MailLockType lock_type,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen                  int try_lock)
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen{
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen    int ret, fd_lock_type;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    /* shared -> exclusive isn't allowed without try_lock */
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    i_assert(try_lock || lock_type != MAIL_LOCK_EXCLUSIVE ||
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen         index->lock_type != MAIL_LOCK_SHARED);
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen    if (index->inconsistent) {
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen        /* index is in inconsistent state and nothing else than
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen           free() is allowed for it. */
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen        return FALSE;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    }
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen    fd_lock_type = MAIL_LOCK_TO_FLOCK(lock_type);
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    if (try_lock) {
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen        ret = file_try_lock(index->fd, fd_lock_type);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen        if (ret < 0)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen            index_set_syscall_error(index, "file_try_lock()");
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen        if (ret <= 0)
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen            return FALSE;
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen    } else {
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen        if (file_wait_lock(index->fd, fd_lock_type,
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen                   DEFAULT_LOCK_TIMEOUT) <= 0) {
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen            return index_set_syscall_error(index,
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen                               "file_wait_lock()");
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen        }
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    }
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen    index->lock_type = lock_type;
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen    debug_mprotect(index->mmap_base, index->mmap_full_length, index);
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen    if (!mail_index_mmap_update(index)) {
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen        (void)index->set_lock(index, MAIL_LOCK_UNLOCK);
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen        return FALSE;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen    }
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen    if (index->indexid != index->header->indexid) {
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen        /* index was rebuilt, there's no way we can maintain
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen           consistency */
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        index_set_error(index, "Warning: Inconsistency - Index "
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                "%s was rebuilt while we had it open",
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                index->filepath);
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        index->inconsistent = TRUE;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        return FALSE;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen    }
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen    if (index->header->flags & MAIL_INDEX_FLAG_FSCK) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        /* someone just partially updated the index, need to fsck it */
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        if (lock_type == MAIL_LOCK_SHARED) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            /* we need exclusive lock so fsck()'s set_lock() won't
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen               get us back here */
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            if (!mail_index_lock_remove(index))
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                return FALSE;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            if (file_wait_lock(index->fd, MAIL_LOCK_EXCLUSIVE,
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                       DEFAULT_LOCK_TIMEOUT) <= 0)
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                return index_set_syscall_error(index,
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                            "file_wait_lock()");
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            index->lock_type = MAIL_LOCK_EXCLUSIVE;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        }
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        /* check again, in case it was already fscked while we had
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen           it unlocked for a while */
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        if (index->header->flags & MAIL_INDEX_FLAG_FSCK) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            if (!index->fsck(index))
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen                return FALSE;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        }
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen        if (lock_type == MAIL_LOCK_SHARED) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            /* drop exclusive lock */
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen            return index->set_lock(index, lock_type);
71da447014454c84828d9dface77219875554d7dTimo Sirainen        }
71da447014454c84828d9dface77219875554d7dTimo Sirainen    }
71da447014454c84828d9dface77219875554d7dTimo Sirainen
71da447014454c84828d9dface77219875554d7dTimo Sirainen    if (lock_type == MAIL_LOCK_EXCLUSIVE) {
71da447014454c84828d9dface77219875554d7dTimo Sirainen        /* while holding exclusive lock, keep the FSCK flag on.
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen           when the lock is released, the FSCK flag will also be
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen           removed. */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index->header->flags |= MAIL_INDEX_FLAG_FSCK;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen        if (!mail_index_fmdatasync(index, sizeof(MailIndexHeader))) {
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen            (void)index->set_lock(index, MAIL_LOCK_UNLOCK);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen            return FALSE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen    return TRUE;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenstatic int mail_index_lock_full(MailIndex *index, MailLockType lock_type,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                int try_lock)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    int keep_fsck;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (index->lock_type == lock_type)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return TRUE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (index->anon_mmap) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* anonymous mmaps are private and don't need any locking */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        mail_index_update_header_changes(index);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index->lock_type = lock_type;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        debug_mprotect(index->mmap_base, index->mmap_full_length,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                   index);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return TRUE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        if (index->modifylog != NULL)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen            mail_modifylog_notify_lock_drop(index->modifylog);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* dropping exclusive lock (either unlock or to shared) */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        keep_fsck = (index->set_flags & MAIL_INDEX_FLAG_FSCK) != 0;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        mail_index_update_header_changes(index);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* remove the FSCK flag only after successful fsync() */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        if (mail_index_sync_file(index) && !keep_fsck) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen            index->header->flags &= ~MAIL_INDEX_FLAG_FSCK;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen            if (msync(index->mmap_base, sizeof(MailIndexHeader),
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                  MS_SYNC) < 0) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                /* we only failed to remove the fsck flag,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                   so this isn't fatal. */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                index_set_syscall_error(index, "msync()");
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen            }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        mail_index_update_timestamp(index);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (lock_type == MAIL_LOCK_UNLOCK)
2d2ebe91d56e9a158de000c9d0026f65600fbcfaTimo Sirainen        return mail_index_lock_remove(index);
2d2ebe91d56e9a158de000c9d0026f65600fbcfaTimo Sirainen    else
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return mail_index_lock_change(index, lock_type, try_lock);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenint mail_index_set_lock(MailIndex *index, MailLockType lock_type)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    return mail_index_lock_full(index, lock_type, FALSE);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenint mail_index_try_lock(MailIndex *index, MailLockType lock_type)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    return mail_index_lock_full(index, lock_type, TRUE);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenint mail_index_verify_hole_range(MailIndex *index)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    MailIndexHeader *hdr;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    unsigned int max_records;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    hdr = index->header;
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen    if (hdr->first_hole_records == 0)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return TRUE;
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen    max_records = MAIL_INDEX_RECORD_COUNT(index);
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen    if (hdr->first_hole_index >= max_records) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index_set_corrupted(index,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                    "first_hole_index points outside file");
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return FALSE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    /* check that first_hole_records is in valid range */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (max_records - hdr->first_hole_index < hdr->first_hole_records) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index_set_corrupted(index,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                    "first_hole_records points outside file");
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return FALSE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    return TRUE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo SirainenMailIndexHeader *mail_index_get_header(MailIndex *index)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    return index->header;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
d78be924ad150840e018eda6a8a7d5e46a39bda2Timo SirainenMailIndexRecord *mail_index_lookup(MailIndex *index, unsigned int seq)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    MailIndexHeader *hdr;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    MailIndexRecord *rec;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    const char *format;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    unsigned int idx;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    i_assert(seq > 0);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
d78be924ad150840e018eda6a8a7d5e46a39bda2Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    hdr = index->header;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (seq > hdr->messages_count) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* out of range */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return NULL;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (!mail_index_verify_hole_range(index))
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return NULL;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    idx = seq-1;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (hdr->first_hole_records == 0 || hdr->first_hole_index > idx) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* easy, it's just at the expected index */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        format = "Invalid first_hole_index in header: %"PRIuUOFF_T;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    } else if (hdr->first_hole_records ==
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen           MAIL_INDEX_RECORD_COUNT(index) - hdr->messages_count) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* only one hole in file, skip it and we're at
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen           correct position */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        idx += hdr->first_hole_records;
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen        format = "Invalid hole locations in header: %"PRIuUOFF_T;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    } else {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        /* find from binary tree */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        idx = mail_tree_lookup_sequence(index->tree, seq);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        if (idx == (unsigned int)-1) {
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen            index_set_corrupted(index, "Sequence %u not found from "
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen                        "binary tree (%u msgs says header)",
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen                        seq, hdr->messages_count);
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen            return NULL;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen        format = "Invalid offset returned by binary tree: %"PRIuUOFF_T;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (idx >= MAIL_INDEX_RECORD_COUNT(index)) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index_set_corrupted(index, format, idx);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return NULL;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    rec = (MailIndexRecord *) ((char *) index->mmap_base +
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                   sizeof(MailIndexHeader)) + idx;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (rec->uid == 0) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        index_set_corrupted(index, format, idx);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return NULL;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    return rec;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo SirainenMailIndexRecord *mail_index_next(MailIndex *index, MailIndexRecord *rec)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    MailIndexRecord *end_rec;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    i_assert(rec >= (MailIndexRecord *) index->mmap_base);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    if (rec == NULL)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        return NULL;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    /* go to the next non-deleted record */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    end_rec = (MailIndexRecord *) ((char *) index->mmap_base +
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen                       index->mmap_used_length);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    while (++rec < end_rec) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen        if (rec->uid != 0)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen            return rec;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen    }
71da447014454c84828d9dface77219875554d7dTimo Sirainen
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen    return NULL;
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen}
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo SirainenMailIndexRecord *mail_index_lookup_uid_range(MailIndex *index,
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen                         unsigned int first_uid,
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen                         unsigned int last_uid,
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen                         unsigned int *seq_r)
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen{
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen    MailIndexRecord *rec;
904ca86f5d3f27505f50f5342214aabb78629cc8Timo Sirainen    unsigned int idx;
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen    i_assert(first_uid > 0 && last_uid > 0);
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen    i_assert(first_uid <= last_uid);
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen
0c1835a90dd1dcedaeaedd1cd91672299cbeb5beTimo Sirainen    idx = mail_tree_lookup_uid_range(index->tree, seq_r,
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen                     first_uid, last_uid);
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen    if (idx == (unsigned int)-1)
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen        return NULL;
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen    if (idx >= MAIL_INDEX_RECORD_COUNT(index)) {
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen        index_set_error(index, "Corrupted binary tree for index %s: "
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen                "lookup returned index outside range "
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen                "(%u >= %"PRIuSIZE_T")", index->filepath, idx,
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen                MAIL_INDEX_RECORD_COUNT(index));
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen        index->set_flags |= MAIL_INDEX_FLAG_REBUILD_TREE;
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen        return NULL;
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen    }
f4735bf7ec2019fdc730e9ebdb39e5a4ea580405Timo Sirainen
    rec = (MailIndexRecord *) ((char *) index->mmap_base +
                   sizeof(MailIndexHeader)) + idx;
    if (rec->uid < first_uid || rec->uid > last_uid) {
        index_set_error(index, "Corrupted binary tree for index %s: "
                "lookup returned offset to wrong UID "
                "(%u vs %u..%u)", index->filepath,
                rec->uid, first_uid, last_uid);
        index->set_flags |= MAIL_INDEX_FLAG_REBUILD_TREE;
        return NULL;
    }

    return rec;
}

const char *mail_index_lookup_field(MailIndex *index, MailIndexRecord *rec,
                    MailDataField field)
{
    MailIndexDataRecord *datarec;

    datarec = (rec->data_fields & field) == 0 ? NULL :
        mail_index_data_lookup(index->data, rec, field);
    if (datarec == NULL)
        return NULL;

    if (!mail_index_data_record_verify(index->data, datarec)) {
        /* index is corrupted, it will be rebuilt */
        return NULL;
    }

    return datarec->data;
}

const void *mail_index_lookup_field_raw(MailIndex *index, MailIndexRecord *rec,
                    MailDataField field, size_t *size)
{
    MailIndexDataRecordHeader *datahdr;
    MailIndexDataRecord *datarec;

    if ((rec->data_fields & field) == 0) {
        *size = 0;
        return NULL;
    }

    if (field < DATA_FIELD_LAST) {
        /* read data field */
        datarec = mail_index_data_lookup(index->data, rec, field);
        if (datarec == NULL) {
            *size = 0;
            return NULL;
        }

        *size = datarec->full_field_size;
        return datarec->data;
    }

    /* read header field */
    datahdr = mail_index_data_lookup_header(index->data, rec);
    if (datahdr == NULL) {
        *size = 0;
        return NULL;
    }

    switch (field) {
    case DATA_HDR_INTERNAL_DATE:
        *size = sizeof(datahdr->internal_date);
        return &datahdr->internal_date;
    case DATA_HDR_VIRTUAL_SIZE:
        *size = sizeof(datahdr->virtual_size);
        return &datahdr->virtual_size;
    case DATA_HDR_HEADER_SIZE:
        *size = sizeof(datahdr->header_size);
        return &datahdr->header_size;
    case DATA_HDR_BODY_SIZE:
        *size = sizeof(datahdr->body_size);
        return &datahdr->body_size;
    default:
        *size = 0;
        return NULL;
    }
}

void mail_index_cache_fields_later(MailIndex *index, MailDataField field)
{
    i_assert(index->lock_type != MAIL_LOCK_UNLOCK);

    field &= ~index->never_cache_fields;

    /* first check if the field even could be in the file */
    if ((index->set_cache_fields & field) != field) {
        if ((index->header->cache_fields & field) == 0) {
            /* no, but make sure the future records will have it.
               we don't immediately mark the index to cache this
               field for old messages as some clients never ask
               the info again */
            index->set_cache_fields |= field;
        } else {
            /* this is at least the second time it's being asked,
               make sure it'll be cached soon. */
            index->set_flags |= MAIL_INDEX_FLAG_CACHE_FIELDS;
        }
    }
}

time_t mail_get_internal_date(MailIndex *index, MailIndexRecord *rec)
{
    const time_t *date;
    size_t size;

    date = index->lookup_field_raw(index, rec,
                       DATA_HDR_INTERNAL_DATE, &size);
    if (date == NULL)
        return (time_t)-1;
    else {
        i_assert(size == sizeof(*date));
        return *date;
    }
}

void mail_index_mark_flag_changes(MailIndex *index, MailIndexRecord *rec,
                  MailFlags old_flags, MailFlags new_flags)
{
    if ((old_flags & MAIL_SEEN) == 0 && (new_flags & MAIL_SEEN)) {
        /* unseen -> seen */
        index->header->seen_messages_count++;
    } else if ((old_flags & MAIL_SEEN) && (new_flags & MAIL_SEEN) == 0) {
        /* seen -> unseen */
        if (index->header->seen_messages_count ==
            index->header->messages_count) {
            /* this is the first unseen message */
                        index->header->first_unseen_uid_lowwater = rec->uid;
        } else if (rec->uid < index->header->first_unseen_uid_lowwater)
            index->header->first_unseen_uid_lowwater = rec->uid;

        if (index->header->seen_messages_count == 0) {
            index_set_corrupted(index, "seen_messages_count in "
                        "header is invalid");
        } else {
            index->header->seen_messages_count--;
        }
    }

    if ((old_flags & MAIL_DELETED) == 0 &&
           (new_flags & MAIL_DELETED)) {
        /* undeleted -> deleted */
        index->header->deleted_messages_count++;

        if (index->header->deleted_messages_count == 1) {
            /* this is the first deleted message */
            index->header->first_deleted_uid_lowwater = rec->uid;
        } else if (rec->uid < index->header->first_deleted_uid_lowwater)
            index->header->first_deleted_uid_lowwater = rec->uid;
    } else if ((old_flags & MAIL_DELETED) &&
           (new_flags & MAIL_DELETED) == 0) {
        /* deleted -> undeleted */
        if (index->header->deleted_messages_count == 0) {
            index_set_corrupted(index, "deleted_messages_count in "
                        "header is invalid");
        } else {
            index->header->deleted_messages_count--;
        }
    }
}

static void update_first_hole_records(MailIndex *index)
{
        MailIndexRecord *rec, *end_rec;

    /* see if first_hole_records can be grown */
    rec = (MailIndexRecord *) ((char *) index->mmap_base +
                   sizeof(MailIndexHeader)) +
        index->header->first_hole_index +
        index->header->first_hole_records;
    end_rec = (MailIndexRecord *) ((char *) index->mmap_base +
                       index->mmap_used_length);
    while (rec < end_rec && rec->uid == 0) {
        index->header->first_hole_records++;
        rec++;
    }
}

static int mail_index_truncate_hole(MailIndex *index)
{
    index->header->used_file_size = sizeof(MailIndexHeader) +
        (uoff_t)index->header->first_hole_index *
        sizeof(MailIndexRecord);
    index->header->first_hole_index = 0;
    index->header->first_hole_records = 0;

    index->mmap_used_length = index->header->used_file_size;
    if (!mail_index_truncate(index))
        return FALSE;

    if (index->header->messages_count == 0) {
        /* all mail was deleted, truncate data file */
        if (!mail_index_data_reset(index->data))
            return FALSE;
    }

    return TRUE;
}

#define INDEX_NEED_COMPRESS(records, hdr) \
    ((records) > INDEX_MIN_RECORDS_COUNT && \
     (records) * (100-INDEX_COMPRESS_PERCENTAGE) / 100 > \
        (hdr)->messages_count)

int mail_index_expunge(MailIndex *index, MailIndexRecord *rec,
               unsigned int seq, int external_change)
{
    MailIndexHeader *hdr;
    unsigned int records, uid, idx;

    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
    i_assert(seq != 0);
    i_assert(rec->uid != 0);

    if (!mail_index_verify_hole_range(index))
        return FALSE;

    hdr = index->header;

    /* setting UID to 0 is enough for deleting the mail from index */
    uid = rec->uid;
    rec->uid = 0;

    /* update first hole */
    idx = INDEX_RECORD_INDEX(index, rec);
    if (hdr->first_hole_records == 0) {
        /* first deleted message in index */
        hdr->first_hole_index = idx;
        hdr->first_hole_records = 1;
    } else if (idx+1 == hdr->first_hole_index) {
        /* deleted the previous record before hole */
        hdr->first_hole_index--;
        hdr->first_hole_records++;
    } else if (idx == hdr->first_hole_index + hdr->first_hole_records) {
        /* deleted the next record after hole */
        hdr->first_hole_records++;
        update_first_hole_records(index);
    } else {
        /* second hole coming to index file */
        if (idx < hdr->first_hole_index) {
            /* new hole before the old hole */
            hdr->first_hole_index = idx;
            hdr->first_hole_records = 1;
        }
    }

    /* update message counts */
    if (hdr->messages_count == 0) {
        /* corrupted */
        index_set_corrupted(index, "Header says there's no mail "
                    "while expunging");
        return FALSE;
    }

    hdr->messages_count--;
    mail_index_mark_flag_changes(index, rec, rec->msg_flags, 0);

    (void)mail_index_data_delete(index->data, rec);

    records = MAIL_INDEX_RECORD_COUNT(index);
    if (hdr->first_hole_index + hdr->first_hole_records == records) {
        /* the hole reaches end of file, truncate it */
        (void)mail_index_truncate_hole(index);
    } else {
        if (INDEX_NEED_COMPRESS(records, hdr))
            hdr->flags |= MAIL_INDEX_FLAG_COMPRESS;
    }

    /* expunge() may be called while index is being rebuilt and when
       tree file hasn't been opened yet */
    if (index->tree != NULL)
        mail_tree_delete(index->tree, uid);
    else {
        /* make sure it also gets updated */
        index->header->flags |= MAIL_INDEX_FLAG_REBUILD_TREE;
    }

    if (seq != 0 && index->modifylog != NULL) {
        if (!mail_modifylog_add_expunge(index->modifylog, seq,
                        uid, external_change))
            return FALSE;
    }

    return TRUE;
}

int mail_index_update_flags(MailIndex *index, MailIndexRecord *rec,
                unsigned int seq, MailFlags flags,
                int external_change)
{
    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
    i_assert(seq != 0);

    if (flags == rec->msg_flags)
        return TRUE; /* no changes */

        mail_index_mark_flag_changes(index, rec, rec->msg_flags, flags);

    rec->msg_flags = flags;
    return index->modifylog == NULL ? TRUE :
        mail_modifylog_add_flags(index->modifylog, seq,
                     rec->uid, external_change);
}

static int mail_index_grow(MailIndex *index)
{
    uoff_t pos;
    unsigned int grow_count;
    void *base;

    grow_count = index->header->messages_count *
        INDEX_GROW_PERCENTAGE / 100;
    if (grow_count < 16)
        grow_count = 16;

    pos = index->mmap_full_length + (grow_count * sizeof(MailIndexRecord));
    i_assert(pos < OFF_T_MAX);

    if (index->anon_mmap) {
        i_assert(pos < SSIZE_T_MAX);

        base = mremap_anon(index->mmap_base, index->mmap_full_length,
                   (size_t)pos, MREMAP_MAYMOVE);
        if (base == MAP_FAILED)
            return index_set_syscall_error(index, "mremap_anon()");

        index->mmap_base = base;
        index->mmap_full_length = (size_t)pos;
        return mmap_verify(index);
    }

    if (file_set_size(index->fd, (off_t)pos) < 0) {
        if (errno == ENOSPC)
            index->nodiskspace = TRUE;
        return index_set_syscall_error(index, "file_set_size()");
    }

    /* file size changed, let others know about it too by changing
       sync_id in header. */
    index->header->sync_id++;

    if (!mail_index_mmap_update(index))
        return FALSE;

    return TRUE;
}

MailIndexRecord *mail_index_append_begin(MailIndex *index)
{
    MailIndexRecord *rec;

    i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);

    if (index->mmap_used_length == index->mmap_full_length) {
        if (!mail_index_grow(index))
            return NULL;
    }

    i_assert(index->header->used_file_size == index->mmap_used_length);
    i_assert(index->mmap_used_length + sizeof(MailIndexRecord) <=
         index->mmap_full_length);

    rec = (MailIndexRecord *) ((char *) index->mmap_base +
                   index->mmap_used_length);
    memset(rec, 0, sizeof(MailIndexRecord));

    index->header->used_file_size += sizeof(MailIndexRecord);
    index->mmap_used_length += sizeof(MailIndexRecord);

    return rec;
}

int mail_index_append_end(MailIndex *index, MailIndexRecord *rec)
{
    i_assert(rec->uid == 0);

    index->header->messages_count++;
    rec->uid = index->header->next_uid++;

    if (index->tree != NULL) {
        mail_tree_insert(index->tree, rec->uid,
                 INDEX_RECORD_INDEX(index, rec));
    }

    return TRUE;
}

const char *mail_index_get_last_error(MailIndex *index)
{
    return index->error;
}

int mail_index_is_diskspace_error(MailIndex *index)
{
    return !index->inconsistent && index->nodiskspace;
}

int mail_index_is_inconsistency_error(MailIndex *index)
{
    return index->inconsistent;
}