mail-index.c revision c61d52810496376a2ea60b8f4e27bbcaa8754f3f
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "hash.h"
#include "mmap-util.h"
#include "nfs-workarounds.h"
#include "read-full.h"
#include "write-full.h"
#include "mail-index-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-cache.h"
#include <stdio.h>
#include <stddef.h>
#include <time.h>
const struct mail_index_header *hdr);
{
struct mail_index *index;
struct mail_index_registered_ext, 5);
return index;
}
{
}
{
}
{
const struct mail_index_registered_ext *extensions;
struct mail_index_registered_ext rext;
unsigned int i, ext_count;
/* see if it's already there */
for (i = 0; i < ext_count; i++) {
return i;
}
return ext_count;
}
{
struct mail_index_registered_ext *rext;
}
{
struct mail_index_registered_ext *rext;
}
{
struct mail_index_registered_ext *rext;
}
{
struct mail_index_registered_ext *rext;
}
{
}
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
for (i = 0; i < count; i++) {
break;
}
}
}
unsigned int initial_count)
{
#define EXTENSION_NAME_APPROX_LEN 20
sizeof(struct mail_index_ext) +
sizeof(uint32_t));
pool_alloconly_create("extensions",
} else {
}
struct mail_index_ext, initial_count);
}
{
const struct mail_index_ext *extensions;
unsigned int i, size;
return (uint32_t)-1;
for (i = 0; i < size; i++) {
return i;
}
return (uint32_t)-1;
}
{
struct mail_index_ext *ext;
idx = 0;
} else {
}
/* Update index ext_id -> map ext_id mapping. Fill non-used
ext_ids with (uint32_t)-1 */
return idx;
}
{
return FALSE;
return TRUE;
}
{
}
struct mail_index_map *map)
{
const struct mail_index_ext_header *ext_hdr;
unsigned int i, old_count;
const char *name;
/* extension headers always start from 64bit offsets, so if base header
doesn't happen to be 64bit aligned we'll skip some bytes */
/* nothing to do, skip allocatations and all */
return 1;
}
for (i = 0; i < old_count; i++)
/* Extension header contains:
- struct mail_index_ext_header
- name (not 0-terminated)
- 64bit alignment padding
- extension header contents
- 64bit alignment padding
*/
"Header extension goes outside header",
return -1;
}
t_push();
"Duplicate header extension %s",
t_pop();
return -1;
}
"Record field %s alignmentation %u not used",
t_pop();
return -1;
}
t_pop();
}
return 1;
}
const char *keyword, bool autocreate,
unsigned int *idx_r)
{
char *keyword_dup;
void *value;
/* keywords_hash keeps a name => index mapping of keywords.
Keywords are never removed from it, so the index values are valid
for the lifetime of the mail_index. */
return TRUE;
}
if (!autocreate) {
*idx_r = (unsigned int)-1;
return FALSE;
}
return TRUE;
}
struct mail_index_map *map)
{
const struct mail_index_ext *ext;
const struct mail_index_keyword_header *kw_hdr;
const struct mail_index_keyword_header_rec *kw_rec;
const char *name;
unsigned int i, name_area_end_offset, old_count;
return 0;
}
/* Extension header contains:
- struct mail_index_keyword_header
- struct mail_index_keyword_header_rec * keywords_count
- const char names[] * keywords_count
*/
/* Keywords can only be added into same mapping. Removing requires a
new mapping (recreating the index file) */
/* nothing changed */
return 0;
}
/* make sure the header is valid */
"Keywords removed unexpectedly",
return -1;
}
"keywords_count larger than header size",
return -1;
}
for (i = 0; i < kw_hdr->keywords_count; i++) {
"name_offset points outside allocated header",
return -1;
}
}
"Keyword header doesn't end with NUL",
return -1;
}
/* create file -> index mapping */
unsigned int, kw_hdr->keywords_count);
}
#ifdef DEBUG
/* Check that existing headers are still the same. It's behind DEBUG
since it's pretty useless waste of CPU normally. */
const unsigned int *old_idx;
unsigned int idx;
"Keywords changed unexpectedly",
return -1;
}
}
#endif
/* Register the newly seen keywords */
for (; i < kw_hdr->keywords_count; i++) {
unsigned int idx;
}
return 0;
}
{
/* Make sure all the keywords are in index->keywords. It's quick to do
if nothing has changed. */
}
struct mail_index_map *map)
{
enum mail_index_header_compat_flags compat_flags = 0;
#ifndef WORDS_BIGENDIAN
#endif
/* major version change - handle silently(?) */
return -1;
}
/* architecture change - handle silently(?) */
return -1;
}
/* we've already complained about it */
return -1;
}
/* following some extra checks that only take a bit of CPU */
"uid_validity = 0, next_uid = %u",
return -1;
}
"record_size too small: %u < %"PRIuSIZE_T,
sizeof(struct mail_index_record));
return -1;
}
return 0;
return 0;
return 0;
return 0;
}
struct mail_index_map *map)
{
}
map->mmap_used_size = 0;
map->records_count = 0;
}
}
{
return;
}
const struct mail_index_header *hdr)
{
/* header smaller than ours, make a copy so our newer headers
won't have garbage in them */
} else {
}
}
{
const struct mail_index_header *hdr;
unsigned int records_count;
/* we had temporarily used a buffer, eg. for updating index */
}
return -1;
}
/* major version change - handle silently */
return 0;
}
return 0;
}
"messages_count too large (%u > %u)",
return 0;
}
return 1;
}
{
int ret;
/* try to read the whole header, but it's not necessarily an error to
read less since the older versions of the index format could be
smaller. Request reading up to buf_size, but accept if we only got
the header. */
pos = 0;
do {
if (ret > 0)
return ret;
}
static int
{
const struct mail_index_header *hdr;
unsigned char buf[512];
/* major version change - handle silently */
return 0;
}
"Corrupted header sizes (base %u, full %u)",
hdr->header_size);
return 0;
}
/* place the base header into memory. */
/* @UNSAFE: read the rest of the header into memory */
hdr->header_size -
pos);
}
}
if (ret > 0) {
/* header read, read the records now. */
}
/* @UNSAFE */
hdr->header_size);
}
if (ret < 0) {
/* a new index file was renamed over this one. */
return 0;
}
return -1;
}
if (ret == 0) {
"Corrupted index file %s: File too small",
return 0;
}
return 1;
}
struct mail_index_map *map)
{
&prev_offset);
}
struct mail_index_map **map,
bool sync_to_index)
{
struct mail_index_view *view;
struct mail_transaction_log_view *log_view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
int ret;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
/* nothing to do */
return 1;
}
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
/* too much trouble to get this right. */
return 0;
}
} else {
/* sync everything there is */
}
MAIL_TRANSACTION_TYPE_MASK) <= 0) {
/* can't use it. sync by re-reading index. */
return 0;
}
&skipped)) > 0) {
continue;
}
ret = 0;
break;
}
}
ret = 1;
&prev_offset);
if (sync_to_index && ret > 0) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
}
return ret;
}
struct mail_index_map **map,
bool sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret;
bool retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
}
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
if (ret != 0)
return ret;
}
/* notify all "sync lost" handlers */
for (i = 0; i < count; i++)
for (i = 0;; i++) {
return ret;
/* ESTALE - reopen index file */
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
}
return -1;
}
}
/* Too many ESTALE retries */
return -1;
}
{
const struct mail_index_header *hdr;
return 0;
/* always check corrupted-flag to avoid errors later */
return -1;
return 1;
}
return 0;
}
{
struct mail_index_map *map;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
return 1;
}
if (ret != 0) {
return ret;
}
/* we're syncing, don't break the mapping */
return 1;
}
}
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
}
} else {
}
map->hdr_copy_buf =
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
}
if (!index->mmap_disable)
else
if (ret > 0) {
if (ret < 0)
ret = 0;
else if (ret == 0) {
ret = 1;
}
}
if (ret <= 0) {
return ret;
}
return 1;
}
struct mail_index_header *hdr_r)
{
unsigned int i;
int ret;
if (!index->mmap_disable) {
if (ret > 0)
else
return ret;
}
for (i = 0;; i++) {
&pos);
break;
/* ESTALE - reopen index file */
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
}
return -1;
}
}
if (ret < 0)
return ret;
}
struct mail_index_map *
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
unsigned int i, count;
else {
for (i = 0; i < map->records_count; i++) {
}
}
/* if we're syncing transaction log into memory and later use the
mapping for updating the index, we need to remember what has
changed */
if (map->write_to_disk) {
}
/* copy extensions */
/* fix the name pointers to use our own pool */
for (i = 0; i < count; i++) {
extensions[i].name);
}
}
return mem_map;
}
{
return 0;
}
{
/* Note that our caller must close index->fd by itself.
mail_index_reopen() for example wants to revert back to old
index file if opening the new one fails. */
}
/* have to create it */
return 0;
}
return 1;
}
static int
{
unsigned int lock_id;
int ret;
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (ret <= 0)
return ret;
return -1;
}
if (ret == 0) {
/* it's corrupted - recreate it */
*lock_id_r = 0;
} else {
else
}
return ret;
}
const struct mail_index_header *hdr)
{
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
"pwrite_full()");
return -1;
}
}
}
return 0;
}
{
const char *path;
int fd;
if (fd == -1)
return -1;
}
return fd;
}
struct mail_index_header *hdr)
{
const char *path;
int ret;
/* log file lock protects index creation */
return -1;
if (ret != 0) {
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
/* create it fully in index.tmp first */
ret = -1;
ret = -1;
} else {
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
"unlink()");
}
} else {
/* make it visible to others */
ret = -1;
}
}
return ret;
}
{
#ifndef WORDS_BIGENDIAN
#endif
}
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
}
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
}
/* returns -1 = error, 0 = won't create, 1 = ok */
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
if (ret > 0)
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 &&
return 0;
} else if (ret < 0)
return -1;
return -1;
/* looks like someone else created the transaction log
before we had the chance. use its indexid so we
don't try to create conflicting ones. */
}
if (lock_id != 0) {
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
/* fallback to in-memory index */
}
} else {
}
}
if (lock_id == 0) {
return -1;
}
return 1;
}
{
int i = 0, ret;
/* corrupted, reopen files */
} else {
return 1;
}
}
i_strdup("(in-memory index)") :
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == MAIL_INDEX_LOCK_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
if (ret <= 0)
break;
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
return ret;
}
{
}
}
{
struct mail_index_map *old_map;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
index->shared_lock_count = 0;
if (fd != -1) {
ret = 0;
} else {
if (ret > 0)
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
ret = -1;
}
if (lock_id != 0)
if (ret == 0) {
} else {
}
}
return ret;
}
{
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
/* deleted, reopen */
return -1;
return 1;
}
}
return -1;
/* lost it? recreate */
(void)mail_index_mark_corrupted(index);
return -1;
}
return -1;
return 1;
} else {
return 0;
}
}
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
return -1;
return ret <= 0 ? -1 : 0;
}
{
}
{
else {
}
return -1;
}
{
}
{
struct mail_index_map *map;
int ret = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
/* set the index as being into memory */
/* mbox file was never even opened. just mark it as being in
memory and let the caller re-open the index. */
return -1;
}
/* move index map to memory */
/* move transaction log to memory */
ret = -1;
}
/* close the index file. */
return ret;
}
{
struct mail_index_header hdr;
return;
}
}
const char *function)
{
return -1;
}
}
const char *filepath,
const char *function)
{
return -1;
}
}
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
{
}
{
}
}
{
unsigned char buf[4];
offset >>= 2;
}
{
return 0;
}