mail-index.c revision 2b498cb82aaad8a11adb5a27a29c55b9c334a1ec
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "hash.h"
#include "mmap-util.h"
#include "nfs-workarounds.h"
#include "read-full.h"
#include "write-full.h"
#include "mail-index-private.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-cache.h"
#include <stdio.h>
#include <stddef.h>
#include <time.h>
struct mail_index_module_register mail_index_module_register = { 0 };
const struct mail_index_header *hdr);
{
struct mail_index *index;
return index;
}
{
}
{
}
{
const struct mail_index_registered_ext *extensions;
struct mail_index_registered_ext rext;
unsigned int i, ext_count;
/* see if it's already there */
for (i = 0; i < ext_count; i++) {
return i;
}
return ext_count;
}
void *context)
{
struct mail_index_registered_ext *rext;
}
{
struct mail_index_registered_ext *rext;
}
{
struct mail_index_registered_ext *rext;
}
{
struct mail_index_registered_ext *rext;
}
{
}
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
for (i = 0; i < count; i++) {
break;
}
}
}
unsigned int initial_count)
{
#define EXTENSION_NAME_APPROX_LEN 20
#define EXT_GLOBAL_ALLOC_SIZE \
#define EXT_PER_ALLOC_SIZE \
sizeof(struct mail_index_ext) + sizeof(uint32_t))
pool_alloconly_create("map extensions",
} else {
/* try to use the existing pool's size for initial_count so
we don't grow it unneededly */
}
}
}
{
const struct mail_index_ext *extensions;
unsigned int i, size;
return (uint32_t)-1;
for (i = 0; i < size; i++) {
return i;
}
return (uint32_t)-1;
}
{
struct mail_index_ext *ext;
idx = 0;
} else {
}
/* Update index ext_id -> map ext_id mapping. Fill non-used
ext_ids with (uint32_t)-1 */
return idx;
}
{
return FALSE;
return TRUE;
}
{
}
struct mail_index_map *map)
{
const struct mail_index_ext_header *ext_hdr;
unsigned int i, old_count;
const char *name;
/* extension headers always start from 64bit offsets, so if base header
doesn't happen to be 64bit aligned we'll skip some bytes */
/* nothing to do, skip allocatations and all */
return 1;
}
for (i = 0; i < old_count; i++)
/* Extension header contains:
- struct mail_index_ext_header
- name (not 0-terminated)
- 64bit alignment padding
- extension header contents
- 64bit alignment padding
*/
"Header extension goes outside header",
return -1;
}
t_push();
"Duplicate header extension %s",
t_pop();
return -1;
}
"Record field %s points outside record size "
t_pop();
return -1;
}
"Record field %s alignmentation %u not used",
t_pop();
return -1;
}
t_pop();
}
return 1;
}
const char *keyword, bool autocreate,
unsigned int *idx_r)
{
char *keyword_dup;
void *value;
/* keywords_hash keeps a name => index mapping of keywords.
Keywords are never removed from it, so the index values are valid
for the lifetime of the mail_index. */
return TRUE;
}
if (!autocreate) {
*idx_r = (unsigned int)-1;
return FALSE;
}
return TRUE;
}
struct mail_index_map *map)
{
const struct mail_index_ext *ext;
const struct mail_index_keyword_header *kw_hdr;
const struct mail_index_keyword_header_rec *kw_rec;
const char *name;
unsigned int i, name_area_end_offset, old_count;
return 0;
}
/* Extension header contains:
- struct mail_index_keyword_header
- struct mail_index_keyword_header_rec * keywords_count
- const char names[] * keywords_count
*/
/* Keywords can only be added into same mapping. Removing requires a
new mapping (recreating the index file) */
/* nothing changed */
return 0;
}
/* make sure the header is valid */
"Keywords removed unexpectedly",
return -1;
}
"keywords_count larger than header size",
return -1;
}
for (i = 0; i < kw_hdr->keywords_count; i++) {
"name_offset points outside allocated header",
return -1;
}
}
"Keyword header doesn't end with NUL",
return -1;
}
/* create file -> index mapping */
#ifdef DEBUG
/* Check that existing headers are still the same. It's behind DEBUG
since it's pretty useless waste of CPU normally. */
const unsigned int *old_idx;
unsigned int idx;
"Keywords changed unexpectedly",
return -1;
}
}
#endif
/* Register the newly seen keywords */
for (; i < kw_hdr->keywords_count; i++) {
unsigned int idx;
}
return 0;
}
{
/* Make sure all the keywords are in index->keywords. It's quick to do
if nothing has changed. */
}
{
enum mail_index_header_compat_flags compat_flags = 0;
#ifndef WORDS_BIGENDIAN
#endif
/* major version change - handle silently(?) */
return FALSE;
}
/* architecture change - handle silently(?) */
return FALSE;
}
/* we've already complained about it */
return FALSE;
}
return TRUE;
}
struct mail_index_map *map)
{
return -1;
/* following some extra checks that only take a bit of CPU */
"uid_validity = 0, next_uid = %u",
return -1;
}
"record_size too small: %u < %"PRIuSIZE_T,
sizeof(struct mail_index_record));
return -1;
}
return 0;
return 0;
return 0;
return 0;
if (map->records_count > 0) {
/* last message's UID must be smaller than next_uid.
also make sure it's not zero. */
const struct mail_index_record *rec;
return 0;
}
}
struct mail_index_map *map)
{
}
map->mmap_used_size = 0;
map->records_count = 0;
}
}
{
return;
}
const struct mail_index_header *hdr)
{
/* header smaller than ours, make a copy so our newer headers
won't have garbage in them */
} else {
}
}
{
const struct mail_index_header *hdr;
unsigned int records_count;
/* we had temporarily used a buffer, eg. for updating index */
}
return -1;
}
/* major version change - handle silently */
return 0;
}
return 0;
}
if (!mail_index_check_header_compat(hdr)) {
/* Can't use this file */
return 0;
}
"messages_count too large (%u > %u)",
return 0;
}
return 1;
}
{
int ret;
/* try to read the whole header, but it's not necessarily an error to
read less since the older versions of the index format could be
smaller. Request reading up to buf_size, but accept if we only got
the header. */
pos = 0;
do {
if (ret > 0)
return ret;
}
static int
{
const struct mail_index_header *hdr;
unsigned char buf[512];
unsigned int records_count;
/* major version change - handle silently */
return 0;
}
return -1;
}
if (!mail_index_check_header_compat(hdr)) {
/* Can't use this file */
return 0;
}
"Corrupted header sizes (base %u, full %u)",
hdr->header_size);
return 0;
}
return 0;
}
/* place the base header into memory. */
/* @UNSAFE: read the rest of the header into memory */
hdr->header_size -
pos);
}
}
if (ret > 0) {
/* header read, read the records now. */
(hdr->record_size != 0 &&
"messages_count too large (%u > %u)",
return 0;
}
}
/* @UNSAFE */
hdr->header_size);
}
if (ret < 0) {
/* a new index file was renamed over this one. */
return 0;
}
return -1;
}
if (ret == 0) {
"Corrupted index file %s: File too small",
return 0;
}
return 1;
}
struct mail_index_map *map)
{
&prev_offset);
}
struct mail_index_map **map,
bool sync_to_index)
{
struct mail_index_view *view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
int ret;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
/* nothing to do */
return 1;
}
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
/* too much trouble to get this right. */
return 0;
}
} else {
/* sync everything there is */
}
MAIL_TRANSACTION_TYPE_MASK) <= 0) {
/* can't use it. sync by re-reading index. */
return 0;
}
continue;
}
ret = 0;
break;
}
}
ret = 1;
&prev_offset);
if (sync_to_index && ret > 0) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
}
return ret;
}
struct mail_index_map **map,
bool sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret;
bool retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
}
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
if (ret != 0)
return ret;
}
/* notify all "sync lost" handlers */
for (i = 0; i < count; i++)
for (i = 0;; i++) {
return ret;
/* ESTALE - reopen index file */
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
}
return -1;
}
}
}
{
const struct mail_index_header *hdr;
int ret;
return 0;
/* always check corrupted-flag to avoid errors later */
return -1;
/* make sure the header is still valid. it also re-parses
extensions although they shouldn't change without the whole
index being recreated */
if (ret > 0)
return 1;
/* broken. fallback to re-mmaping which will catch it */
}
return 0;
}
{
struct mail_index_map *map;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
return 1;
}
if (ret != 0) {
return ret;
}
/* we're syncing, don't break the mapping */
return 1;
}
}
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
}
} else {
}
map->hdr_copy_buf =
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
}
if (!index->mmap_disable)
else
if (ret > 0) {
if (ret < 0)
ret = 0;
else if (ret == 0) {
ret = 1;
}
}
if (ret <= 0) {
return ret;
}
return 1;
}
struct mail_index_header *hdr_r)
{
unsigned int i;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
return TRUE;
}
if (!index->mmap_disable) {
if (ret > 0)
else
return ret;
}
for (i = 0;; i++) {
&pos);
break;
/* ESTALE - reopen index file */
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
}
return -1;
}
}
if (ret < 0)
return ret;
}
struct mail_index_map *
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
unsigned int i, count;
else {
for (i = 0; i < map->records_count; i++) {
}
}
/* if we're syncing transaction log into memory and later use the
mapping for updating the index, we need to remember what has
changed */
if (map->write_to_disk) {
}
/* copy extensions */
/* fix the name pointers to use our own pool */
for (i = 0; i < count; i++) {
extensions[i].name);
}
}
return mem_map;
}
{
return 0;
}
{
/* Note that our caller must close index->fd by itself.
mail_index_reopen() for example wants to revert back to old
index file if opening the new one fails. */
}
/* have to create it */
return 0;
}
return 1;
}
static int
{
unsigned int lock_id;
int ret;
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (ret <= 0)
return ret;
return -1;
}
if (ret == 0) {
/* it's corrupted - recreate it */
*lock_id_r = 0;
} else {
else
}
return ret;
}
const struct mail_index_header *hdr)
{
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
"pwrite_full()");
return -1;
}
}
}
return 0;
}
{
const char *path;
int fd;
if (fd == -1)
return -1;
}
return fd;
}
struct mail_index_header *hdr)
{
const char *path;
int ret;
/* log file lock protects index creation */
return -1;
if (ret != 0) {
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
/* create it fully in index.tmp first */
ret = -1;
ret = -1;
} else {
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
"unlink()");
}
} else {
/* make it visible to others */
ret = -1;
}
}
return ret;
}
{
#ifndef WORDS_BIGENDIAN
#endif
}
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
}
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
}
/* returns -1 = error, 0 = won't create, 1 = ok */
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
if (ret > 0)
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 &&
return 0;
} else if (ret < 0)
return -1;
if (ret == 0)
return -1;
}
/* index->indexid may be updated by transaction log opening,
in case someone else had already created a new log file */
if (lock_id != 0) {
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
/* fallback to in-memory index */
}
} else {
}
}
if (lock_id == 0) {
return -1;
}
return 1;
}
enum file_lock_method lock_method)
{
int i = 0, ret;
/* corrupted, reopen files */
} else {
return 1;
}
}
i_strdup("(in-memory index)") :
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
(flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
(flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == FILE_LOCK_METHOD_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
if (ret <= 0)
break;
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
return ret;
}
{
}
}
{
struct mail_index_map *old_map;
struct file_lock *old_file_lock;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
index->shared_lock_count = 0;
if (fd != -1) {
ret = 0;
} else {
if (ret > 0)
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
ret = -1;
}
if (lock_id != 0)
if (ret == 0) {
if (old_file_lock != NULL)
} else {
}
}
return ret;
}
{
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
/* deleted, reopen */
return -1;
return 1;
}
}
return -1;
/* lost it? recreate later */
return -1;
}
return -1;
return 1;
} else {
return 0;
}
}
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
return -1;
return ret <= 0 ? -1 : 0;
}
{
}
{
else {
}
return -1;
}
{
}
{
struct mail_index_map *map;
int ret = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
/* set the index as being into memory */
/* index was never even opened. just mark it as being in
memory and let the caller re-open the index. */
return -1;
}
/* move index map to memory */
}
/* move transaction log to memory */
ret = -1;
}
/* close the index file. */
return ret;
}
{
struct mail_index_header hdr;
return;
}
}
const char *function)
{
return -1;
}
}
const char *filepath,
const char *function)
{
return -1;
}
}
{
}
{
}
}
#ifdef WORDS_BIGENDIAN
/* FIXME: Unfortunately these functions were originally written to use
endian-specific code and we can't avoid that without breaking backwards
compatibility. When we do break it, just select one of these. */
{
offset >>= 2;
}
{
return 0;
return (((offset & 0x0000007f)) |
}
#else
{
offset >>= 2;
}
{
return 0;
}
#endif