src/lib-index/mail-index-open.c

	mail-index-open.c revision 73db234f79f77142305b2238d283d97bfc6b8906
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2002 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "lib.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "ioloop.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "file-lock.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "file-set-size.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "hostpid.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "mmap-util.h"
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include "unlink-lockfiles.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "write-full.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "mail-index.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "mail-index-data.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "mail-index-util.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "mail-tree.h"
9d53cff7ba0e2fc5fd907d9194448d5090e1c403Timo Sirainen#include "mail-lockdir.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "mail-modifylog.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen#include "mail-custom-flags.h"
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include <stdio.h>
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include <stdlib.h>
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include <unistd.h>
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen#include <fcntl.h>
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic const char *index_file_prefixes[] =
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	{ "data", "tree", "log", "log.2", NULL };
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic int delete_index(const char *path)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	char tmp[PATH_MAX];
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	int i;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	/* main index */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	if (unlink(path) < 0)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		return FALSE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	for (i = 0; index_file_prefixes[i] != NULL; i++) {
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		if (i_snprintf(tmp, sizeof(tmp), "%s.%s",
51c331377beb4a2acb81aee4d12bc8ef6c496625Timo Sirainen			       path, index_file_prefixes[i]) < 0)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen			return FALSE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		if (unlink(tmp) < 0)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen			return FALSE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		i++;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	}
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	return TRUE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen}
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainenstatic int read_and_verify_header(int fd, MailIndexHeader *hdr,
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen				  int check_version)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	/* read the header */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	if (lseek(fd, 0, SEEK_SET) != 0)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		return FALSE;
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	if (read(fd, hdr, sizeof(MailIndexHeader)) != sizeof(MailIndexHeader))
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		return FALSE;
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen	/* check the compatibility */
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen	return hdr->compat_data[1] == MAIL_INDEX_COMPAT_FLAGS &&
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		hdr->compat_data[2] == sizeof(unsigned int) &&
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen		hdr->compat_data[3] == sizeof(time_t) &&
043c8a96a035379bcba04f487d58457beefdfcaaTimo Sirainen		hdr->compat_data[4] == sizeof(uoff_t) &&
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen		hdr->compat_data[5] == MEM_ALIGN_SIZE &&
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen		(!check_version || hdr->compat_data[0] == MAIL_INDEX_VERSION);
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen}
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen/* Returns TRUE if we're compatible with given index file. May delete the
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainen   file if it's from older version. */
27a44fcfd8d19bffe0f267f20a2b5d3fe7600fddTimo Sirainenstatic int mail_check_compatible_index(MailIndex *index, const char *path)
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen{
d96f86fb881c5b106649e8994ead1052acf24030Timo Sirainen        MailIndexHeader hdr;
	int fd, compatible;

	fd = open(path, O_RDONLY);
	if (fd == -1) {
		if (errno != ENOENT)
			index_file_set_syscall_error(index, path, "open()");
		return FALSE;
	}

	compatible = read_and_verify_header(fd, &hdr, FALSE);
	if (hdr.compat_data[0] != MAIL_INDEX_VERSION) {
		/* version mismatch */
		compatible = FALSE;
		if (hdr.compat_data[0] < MAIL_INDEX_VERSION) {
			/* of older version, we don't need it anymore */
			(void)delete_index(path);
		}
	}

	(void)close(fd);
	return compatible;
}

/* Returns a file name of compatible index */
static const char *mail_find_index(MailIndex *index)
{
	const char *name;
	char path[PATH_MAX];

	hostpid_init();

	/* first try .imap.index-<hostname> */
	name = t_strconcat(INDEX_FILE_PREFIX "-", my_hostname, NULL);
	if (str_path(path, sizeof(path), index->dir, name) == 0 &&
	    mail_check_compatible_index(index, path))
		return name;

	/* then try the generic .imap.index */
	name = INDEX_FILE_PREFIX;
	if (str_path(path, sizeof(path), index->dir, name) == 0 &&
	    mail_check_compatible_index(index, path))
		return name;

	return NULL;
}

static int mail_index_open_init(MailIndex *index, int update_recent)
{
	MailIndexHeader *hdr;

	hdr = index->header;

	/* update \Recent message counters */
	if (update_recent && hdr->last_nonrecent_uid != hdr->next_uid-1) {
		/* keep last_recent_uid to next_uid-1 */
		if (index->lock_type == MAIL_LOCK_SHARED) {
			if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
				return FALSE;
		}

		if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
			return FALSE;

		index->first_recent_uid = index->header->last_nonrecent_uid+1;
		index->header->last_nonrecent_uid = index->header->next_uid-1;
	} else {
		index->first_recent_uid = hdr->last_nonrecent_uid+1;
	}

	if (hdr->next_uid >= MAX_ALLOWED_UID - 1000) {
		/* UID values are getting too high, rebuild index */
		index->set_flags |= MAIL_INDEX_FLAG_REBUILD;
	}

	if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
		/* finally reset the modify log marks, fsck or syncing might
		   have deleted some messages, and since we're only just
		   opening the index, there's no need to remember them */
		if (!mail_modifylog_mark_synced(index->modifylog))
			return FALSE;
	}

	return TRUE;
}

static int index_open_and_fix(MailIndex *index, int update_recent, int fast)
{
	int rebuild;

	if (!mail_index_mmap_update(index))
		return FALSE;

	rebuild = FALSE;

	/* open/create the index files */
	if (!mail_index_data_open(index)) {
		if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) == 0)
			return FALSE;

		/* data file is corrupted, need to rebuild index */
		rebuild = TRUE;
		index->set_flags = 0;

		if (!mail_index_data_create(index))
			return FALSE;
	}

	/* custom flags file needs to be open before
	   rebuilding index */
	if (!mail_custom_flags_open_or_create(index))
		return FALSE;

	if (rebuild || (index->header->flags & MAIL_INDEX_FLAG_REBUILD)) {
		/* index is corrupted, rebuild */
		if (!index->rebuild(index))
			return FALSE;

		/* no inconsistency problems while still opening
		   the index */
		index->inconsistent = FALSE;
	}

	if (!mail_tree_open_or_create(index))
		return FALSE;
	if (!mail_modifylog_open_or_create(index))
		return FALSE;

	if (index->header->flags & MAIL_INDEX_FLAG_FSCK) {
		/* index needs fscking */
		if (!index->fsck(index))
			return FALSE;
	}

	if (!fast && (index->header->flags & MAIL_INDEX_FLAG_COMPRESS)) {
		/* remove deleted blocks from index file */
		if (!mail_index_compress(index))
			return FALSE;
	}

	if (index->header->flags & MAIL_INDEX_FLAG_REBUILD_TREE) {
		if (!mail_tree_rebuild(index->tree))
			return FALSE;
	}

	/* sync ourself - before updating cache and compression which
	   may happen because of this. */
	if (!index->sync_and_lock(index, MAIL_LOCK_SHARED, NULL))
		return FALSE;

	/* we never want to keep shared lock if syncing happens to set it.
	   either exclusive or nothing (NOTE: drop it directly, not through
	   index->set_lock() so mbox lock won't be affected). */
	if (index->lock_type == MAIL_LOCK_SHARED) {
		if (!mail_index_set_lock(index, MAIL_LOCK_UNLOCK))
			return FALSE;
	}

	if (!fast && (index->header->flags & MAIL_INDEX_FLAG_CACHE_FIELDS)) {
		/* need to update cached fields */
		if (!mail_index_update_cache(index))
			return FALSE;
	}

	if (!fast && (index->header->flags & MAIL_INDEX_FLAG_COMPRESS_DATA)) {
		/* remove unused space from index data file.
		   keep after cache_fields which may move data
		   and create unused space.. */
		if (!mail_index_compress_data(index))
			return FALSE;
	}

	if (!mail_index_open_init(index, update_recent))
		return FALSE;

	return TRUE;
}

static int mail_index_verify_header(MailIndex *index, MailIndexHeader *hdr)
{
	/* if index is being created, we'll wait here until it's finished */
	if (!mail_index_wait_lock(index, F_RDLCK))
		return FALSE;

	/* check the compatibility anyway just to be sure */
	if (!read_and_verify_header(index->fd, hdr, TRUE)) {
		index_set_error(index, "Non-compatible index file %s",
				index->filepath);
		(void)mail_index_wait_lock(index, F_UNLCK);
		return FALSE;
	}

	if (!mail_index_wait_lock(index, F_UNLCK))
		return FALSE;

	return TRUE;
}

static int mail_index_open_file(MailIndex *index, const char *path,
				int update_recent, int fast)
{
        MailIndexHeader hdr;

	/* the index file should already be checked that it exists and
	   we're compatible with it. */

	index->fd = open(path, O_RDWR);
	if (index->fd == -1)
		return index_file_set_syscall_error(index, path, "open()");
	index->filepath = i_strdup(path);

	if (!mail_index_verify_header(index, &hdr)) {
		(void)close(index->fd);
		index->fd = -1;

		i_free(index->filepath);
		index->filepath = NULL;
		return FALSE;
	}

	index->indexid = hdr.indexid;

	/* the shared lock set is just to make sure we drop exclusive lock */
	if (!index_open_and_fix(index, update_recent, fast) ||
	    !index->set_lock(index, MAIL_LOCK_UNLOCK)) {
		mail_index_close(index);
		return FALSE;
	}

	return TRUE;
}

static int mail_index_init_new_file(MailIndex *index, MailIndexHeader *hdr,
				    const char *temp_path)
{
	const char *index_path;
	off_t fsize;

	/* set the index's path temporarily */
	index->filepath = t_strdup_noconst(temp_path);

	if (write_full(index->fd, hdr, sizeof(MailIndexHeader)) < 0) {
		index_set_syscall_error(index, "write_full()");
		index->filepath = NULL;
		return FALSE;
	}

	fsize = sizeof(MailIndexHeader) +
		INDEX_MIN_RECORDS_COUNT * sizeof(MailIndexRecord);
	if (file_set_size(index->fd, fsize) < 0) {
		index_set_syscall_error(index, "file_set_size()");
		index->filepath = NULL;
		return FALSE;
	}

	if (mail_index_wait_lock(index, F_WRLCK) <= 0) {
		index->filepath = NULL;
		return FALSE;
	}
	index->filepath = NULL;

	/* move the temp index into the real one. we also need to figure
	   out what to call ourself on the way. */
	index_path = t_strconcat(index->dir, "/"INDEX_FILE_PREFIX, NULL);
	if (link(temp_path, index_path) == 0) {
		if (unlink(temp_path) < 0) {
			/* doesn't really matter, log anyway */
			index_file_set_syscall_error(index, temp_path,
						     "unlink()");
		}
	} else {
		if (errno != EEXIST) {
			/* fatal error */
			index_set_error(index, "link(%s, %s) failed: %m",
					temp_path, index_path);
			return FALSE;
		}

		if (getenv("OVERWRITE_INCOMPATIBLE_INDEX") != NULL) {
			/* don't try to support different architectures,
			   just overwrite the index if it's already there. */
		} else {
			/* fallback to .imap.index-hostname - we require each
			   system to have a different hostname so it's safe to
			   override previous index as well */
			hostpid_init();

			index_path = t_strconcat(index_path, "-",
						 my_hostname, NULL);
		}

		if (rename(temp_path, index_path) < 0) {
			index_set_error(index, "rename(%s, %s) failed: %m",
					temp_path, index_path);
			return FALSE;
		}
	}

	index->filepath = i_strdup(index_path);
	return TRUE;
}

static int mail_index_create(MailIndex *index, int *dir_unlocked,
			     int update_recent)
{
	MailIndexHeader hdr;
	const char *path;
	int nodiskspace;

	*dir_unlocked = FALSE;

	mail_index_init_header(index, &hdr);

	if (index->nodiskspace) {
		/* don't even bother trying to create it */
	} else {
		/* first create the index into temporary file. */
		index->fd = mail_index_create_temp_file(index, &path);
		if (index->fd != -1) {
			if (!mail_index_init_new_file(index, &hdr, path)) {
				int old_errno = errno;

				(void)close(index->fd);
				(void)unlink(path);
				index->fd = -1;

				errno = old_errno;
			}
		}

		if (index->fd == -1 && errno != ENOSPC) {
			/* fatal failure */
			return FALSE;
		}
	}

	if (index->fd == -1) {
		/* no space for index files, keep it in memory */
		index->mmap_full_length = INDEX_FILE_MIN_SIZE;
		index->mmap_base = mmap_anon(index->mmap_full_length);

		memcpy(index->mmap_base, &hdr, sizeof(hdr));
		index->header = index->mmap_base;
		index->mmap_used_length = index->header->used_file_size;

		index->anon_mmap = TRUE;
		index->filepath = i_strdup("(in-memory index)");
	}

	index->indexid = hdr.indexid;

	/* the fd is actually already locked, now we're just making it
	   clear to the indexing code. */
	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) {
		mail_index_close(index);
		return FALSE;
	}

	/* it's not good to keep the directory locked too long. our index file
	   is locked which is enough. */
	if (!*dir_unlocked && mail_index_lock_dir(index, MAIL_LOCK_UNLOCK))
		*dir_unlocked = TRUE;

	do {
		if (!mail_custom_flags_open_or_create(index))
			break;
		if (!mail_index_data_create(index))
			break;

		nodiskspace = index->nodiskspace;
		if (!index->rebuild(index)) {
			if (!index->anon_mmap && index->nodiskspace) {
				/* we're out of disk space, keep it in
				   memory this time */
				mail_index_close(index);

                                index->nodiskspace = TRUE;
				return mail_index_create(index, dir_unlocked,
							 update_recent);
			}
			break;
		}

		/* rebuild() resets the nodiskspace variable */
		index->nodiskspace = nodiskspace;

		if (!mail_tree_create(index))
			break;
		if (!mail_modifylog_create(index))
			break;

		index->inconsistent = FALSE;

		if (!mail_index_open_init(index, update_recent))
			break;

		if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
			break;

		return TRUE;
	} while (0);

	(void)index->set_lock(index, MAIL_LOCK_UNLOCK);

	mail_index_close(index);
	return FALSE;
}

void mail_index_init_header(MailIndex *index, MailIndexHeader *hdr)
{
	memset(hdr, 0, sizeof(MailIndexHeader));
	hdr->compat_data[0] = MAIL_INDEX_VERSION;
	hdr->compat_data[1] = MAIL_INDEX_COMPAT_FLAGS;
	hdr->compat_data[2] = sizeof(unsigned int);
	hdr->compat_data[3] = sizeof(time_t);
	hdr->compat_data[4] = sizeof(uoff_t);
	hdr->compat_data[5] = MEM_ALIGN_SIZE;
	hdr->indexid = ioloop_time;

	/* mark the index requiring rebuild - rebuild() removes this flag
	   when it succeeds */
	hdr->flags = MAIL_INDEX_FLAG_REBUILD;

	/* set the fields we always want to cache */
	hdr->cache_fields |= index->default_cache_fields;

	hdr->used_file_size = sizeof(MailIndexHeader);
	hdr->uid_validity = ioloop_time;
	hdr->next_uid = 1;
}

static void mail_index_cleanup_temp_files(const char *dir)
{
	unlink_lockfiles(dir, t_strconcat("temp.", my_hostname, NULL),
			 "temp.", time(NULL) - TEMP_FILE_TIMEOUT);
}

void mail_index_init(MailIndex *index, const char *dir)
{
	size_t len;

	index->fd = -1;
	index->dir = i_strdup(dir);

	len = strlen(index->dir);
	if (index->dir[len-1] == '/')
		index->dir[len-1] = '\0';

	index->mail_read_mmaped = getenv("MAIL_READ_MMAPED") != NULL;
}

int mail_index_open(MailIndex *index, int update_recent, int fast)
{
	const char *name, *path;

	i_assert(!index->opened);

	/* this isn't initialized anywhere else */
	index->fd = -1;

	mail_index_cleanup_temp_files(index->dir);

	name = mail_find_index(index);
	if (name == NULL)
		return FALSE;

	path = t_strconcat(index->dir, "/", name, NULL);
	if (!mail_index_open_file(index, path, update_recent, fast))
		return FALSE;

	index->opened = TRUE;
	return TRUE;
}

int mail_index_open_or_create(MailIndex *index, int update_recent, int fast)
{
	int failed, dir_unlocked;

	i_assert(!index->opened);

	mail_index_cleanup_temp_files(index->dir);

	if (mail_index_open(index, update_recent, fast))
	        return TRUE;

	if (index->index_lock_timeout || index->mailbox_lock_timeout)
		return FALSE;

	/* index wasn't found or it was broken. lock the directory and check
	   again, just to make sure we don't end up having two index files
	   due to race condition with another process. */
	if (!mail_index_lock_dir(index, MAIL_LOCK_EXCLUSIVE))
		return FALSE;

	failed = FALSE;
	if (mail_index_open(index, update_recent, fast))
		dir_unlocked = FALSE;
	else if (!index->index_lock_timeout && !index->mailbox_lock_timeout) {
		if (!mail_index_create(index, &dir_unlocked, update_recent))
			failed = TRUE;
	}

	if (!dir_unlocked && !mail_index_lock_dir(index, MAIL_LOCK_UNLOCK))
		return FALSE;

	if (failed)
		return FALSE;

	index->opened = TRUE;
	return TRUE;
}