mail-index-compress.c revision 8d67115ff1d54ba967c835c5aa050111fa54602f
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose/* Copyright (C) 2002 Timo Sirainen */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include "lib.h"
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include "write-full.h"
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include "mail-index.h"
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include "mail-index-data.h"
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include "mail-index-util.h"
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include "mail-hash.h"
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include <stdio.h>
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose#include <unistd.h>
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Boseint mail_index_truncate(MailIndex *index)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose{
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose uoff_t empty_space, truncate_threshold;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* really truncate the file only when it's almost empty */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose empty_space = index->mmap_full_length - index->mmap_used_length;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose truncate_threshold =
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->mmap_full_length / 100 * INDEX_TRUNCATE_PERCENTAGE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (empty_space > truncate_threshold) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->mmap_full_length = index->mmap_used_length +
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose (empty_space * INDEX_TRUNCATE_KEEP_PERCENTAGE / 100);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* keep the size record-aligned */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->mmap_full_length -=
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose (index->mmap_full_length - sizeof(MailIndexHeader)) %
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose sizeof(MailIndexRecord);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return index_set_syscall_error(index, "ftruncate()");
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->sync_id++;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose}
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Boseint mail_index_compress(MailIndex *index)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose{
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose MailIndexRecord *rec, *hole_rec, *end_rec;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (index->header->first_hole_position == 0) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* we don't need to compress after all. shouldn't happen.. */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->flags &= ~MAIL_INDEX_FLAG_COMPRESS;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!mail_index_verify_hole_range(index))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* if we get interrupted, the whole index is probably corrupted.
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose so keep rebuild-flag on while doing this */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->flags |= MAIL_INDEX_FLAG_REBUILD;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!mail_index_fmsync(index, sizeof(MailIndexHeader)))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* first actually compress the data */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose end_rec = (MailIndexRecord *) ((char *) index->mmap_base +
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->mmap_used_length);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose hole_rec = (MailIndexRecord *) ((char *) index->mmap_base +
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->first_hole_position);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose rec = hole_rec + index->header->first_hole_records;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose while (rec < end_rec) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (rec->uid != 0) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose memcpy(hole_rec, rec, sizeof(MailIndexRecord));
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose mail_hash_update(index->hash, rec->uid,
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose INDEX_FILE_POSITION(index, hole_rec));
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose hole_rec++;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose rec++;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* truncate the file to get rid of the extra records */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->mmap_used_length = (size_t) ((char *) hole_rec -
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose (char *) index->mmap_base);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->used_file_size = index->mmap_used_length;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!mail_index_truncate(index))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* update headers */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->first_hole_position = 0;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->first_hole_records = 0;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* make sure the whole file is synced before removing rebuild-flag */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!mail_index_fmsync(index, index->mmap_used_length))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->flags &= ~(MAIL_INDEX_FLAG_COMPRESS |
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose MAIL_INDEX_FLAG_REBUILD);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose}
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bosestatic int mail_index_copy_data(MailIndex *index, int fd, const char *path)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose{
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose MailIndexDataHeader data_hdr;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose MailIndexRecord *rec;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose unsigned char *mmap_data;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose size_t mmap_data_size;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose uoff_t offset;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose mmap_data = mail_index_data_get_mmaped(index->data, &mmap_data_size);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (mmap_data == NULL)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* write data header */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose memset(&data_hdr, 0, sizeof(data_hdr));
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose data_hdr.indexid = index->indexid;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (write_full(fd, &data_hdr, sizeof(data_hdr)) < 0) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (errno == ENOSPC)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->nodiskspace = TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index_set_error(index, "Error writing to temp index data "
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose "%s: %m", path);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* now we'll begin the actual moving. keep rebuild-flag on
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose while doing it. */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->flags |= MAIL_INDEX_FLAG_REBUILD;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!mail_index_fmsync(index, sizeof(MailIndexHeader)))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose offset = sizeof(data_hdr);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose rec = index->lookup(index, 1);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose while (rec != NULL) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (rec->data_position + rec->data_size > mmap_data_size) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index_set_corrupted(index, "data_position+data_size "
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose "points outside file");
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (write_full(fd, mmap_data + rec->data_position,
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose rec->data_size) < 0) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (errno == ENOSPC)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->nodiskspace = TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index_set_error(index, "Error writing to temp index "
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose "data %s: %m", path);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose rec->data_position = offset;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose offset += rec->data_size;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose rec = index->next(index, rec);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose}
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Boseint mail_index_compress_data(MailIndex *index)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose{
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose const char *temppath, *datapath;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose int fd, failed;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (index->anon_mmap)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* write the data into temporary file updating the offsets in index
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose while doing it. if we fail (especially if out of disk space/quota)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose we'll simply fail and index is rebuilt later */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose fd = mail_index_create_temp_file(index, &temppath);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (fd == -1) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (errno == ENOSPC)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->nodiskspace = TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose failed = !mail_index_copy_data(index, fd, temppath);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (close(fd) < 0)
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index_file_set_syscall_error(index, temppath, "close()");
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!failed) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* now, rename the temp file to new data file. but before that
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose reset indexid to make sure that other processes know the
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose data file is closed. */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose (void)mail_index_data_mark_deleted(index->data);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose mail_index_data_free(index->data);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose datapath = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (rename(temppath, datapath) < 0) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (errno == ENOSPC)
b3292840ebaa747a9fd596ff47cc5d18198361d0Michal Zidek index->nodiskspace = TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index_set_error(index, "rename(%s, %s) failed: %m",
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose temppath, datapath);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose failed = TRUE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
1412a7655c53452721d19813b0ab4a1afd2b0744Lukas Slebodnik
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (failed) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (unlink(temppath) < 0) {
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index_file_set_syscall_error(index, temppath,
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose "unlink()");
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose }
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose /* make sure the whole file is synced before removing rebuild-flag */
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose if (!mail_index_fmsync(index, index->mmap_used_length))
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return FALSE;
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose index->header->flags &= ~(MAIL_INDEX_FLAG_COMPRESS_DATA |
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose MAIL_INDEX_FLAG_REBUILD);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose return mail_index_data_open(index);
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose}
caee9828ee30609e9f433957dbb3d0163390a207Sumit Bose