mail-index-strmap.c revision ef5fb27361cc5e15766e85e28355750ff04b13c9
/* Copyright (c) 2008-2009 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "bsearch-insert-pos.h"
#include "istream.h"
#include "ostream.h"
#include "file-lock.h"
#include "file-dotlock.h"
#include "crc32.h"
#include "safe-mkstemp.h"
#include "str.h"
#include "mail-index-private.h"
#include "mail-index-strmap.h"
#include <stdio.h>
struct mail_index_strmap {
struct mail_index *index;
char *path;
int fd;
struct dotlock_settings dotlock_settings;
};
struct mail_index_strmap_view {
struct mail_index_strmap *strmap;
struct mail_index_view *view;
struct hash2_table *hash;
void *cb_context;
unsigned int desynced:1;
};
struct mail_index_strmap_read_context {
struct mail_index_strmap_view *view;
struct mail_index_strmap_rec rec;
unsigned int rec_size;
unsigned int too_large_uids:1;
};
struct mail_index_strmap_view_sync {
struct mail_index_strmap_view *view;
};
struct mail_index_strmap_hash_key {
const char *str;
};
/* number of bytes required to store one string idx */
/* renumber the string indexes when highest string idx becomes larger than
<number of indexes>*STRMAP_FILE_MAX_STRIDX_MULTIPLIER */
#define STRMAP_FILE_MAX_STRIDX_MULTIPLIER 2
#define MAIL_INDEX_STRMAP_TIMEOUT_SECS 10
const struct dotlock_settings default_dotlock_settings = {
};
struct mail_index_strmap *
{
struct mail_index_strmap *strmap;
return strmap;
}
static bool
static void
const char *function)
{
return;
}
"%s failed with strmap index file %s: %m",
}
{
}
}
{
}
static unsigned int mail_index_strmap_hash_key(const void *_key)
{
}
static bool
{
}
struct mail_index_strmap_view *
struct mail_index_view *idx_view,
void *context,
const struct hash2_table **hash_r)
{
struct mail_index_strmap_view *view;
return view;
}
{
}
{
}
{
view->last_added_uid = 0;
view->lost_expunged_uid = 0;
}
static void
{
"Corrupted strmap index file: %s",
}
{
const struct mail_index_header *idx_hdr;
struct mail_index_strmap_header hdr;
const unsigned char *data;
int ret;
return 0;
return -1;
}
if (ret <= 0) {
if (ret < 0) {
} else {
}
return ret;
}
/* need to rebuild. if we already had something in the strmap,
we can keep it. */
return 0;
}
/* we'll read the entire file from the beginning */
view->last_added_uid = 0;
view->last_read_uid = 0;
view->total_ref_count = 0;
return 0;
}
{
/* FIXME: nfs flush */
return TRUE;
}
return TRUE;
}
}
{
return -1;
if (view->lost_expunged_uid != 0) {
/* last read failed because view had a message
that didn't exist in the strmap (because it
was expunged by another session). if the
message still isn't expunged in this view,
just continue using the current strmap. */
return -1;
/* our view isn't synced with the disk, we
can't read strmap without first resetting
the view */
} else {
return 0;
}
}
}
return mail_index_strmap_open(view);
}
static int
{
const unsigned char *data;
int ret;
if (ret <= 0)
return ret;
return -1;
return 1;
}
static int
{
const struct mail_index_record *rec;
/* thread index has larger UIDs than what we've seen
in our view. we'll have to read them again later
when we know about them */
}
return 0;
}
ctx->uid_lookup_idx++;
return 1;
return 0;
} else {
/* record that exists in index is missing from strmap.
see if it's because the strmap is corrupted or because
our current view is a bit stale and the message has already
been expunged. */
return -1;
}
}
static int
{
int ret;
/* <uid> <n> <crc32>*count <str_idx>*count
where
n = 0 -> count=1 (only Message-ID:)
n = 1 -> count=2 (Message-ID: + In-Reply-To:)
n = 2+ -> count=n (Message-ID: + References:)
*/
if (mail_index_strmap_read_packed(ctx, &n) <= 0)
return -1;
if (ret < 0)
return -1;
return -1;
if (ret == 0) {
/* this message has already been expunged, ignore it.
update highest string indexes anyway. */
for (i = 0; i < count; i++) {
}
return 0;
}
/* everything exists. save it. FIXME: these ref_index values
are thread index specific, perhaps something more generic
should be used some day */
ctx->next_ref_index = 0;
i_unreached();
return 1;
}
static bool
{
return FALSE;
}
/* FIXME: str_idx could be stored as packed relative values
(first relative to highest_idx, the rest relative to the
previous str_idx) */
/* read the record contents */
/* get to the next record */
return TRUE;
}
static int
struct mail_index_strmap_read_context *ctx)
{
const unsigned char *data;
int ret;
/* come back later when we know about the new UIDs */
return 0;
}
sizeof(block_size)-1);
if (ret <= 0) {
/* no new data */
return 0;
}
return -1;
}
if (block_size == 0) {
/* the rest of the file is either not written, or the previous
write didn't finish */
return 0;
}
/* block size too large */
return -1;
}
/* FIXME: when reading multiple blocks we shouldn't have to calculate
this every time */
return 1;
}
static int
{
int ret;
return 1;
/* get next UID */
do {
/* this block is done */
return 0;
}
return -1;
} while (ret == 0);
return ret;
}
static int
bool update_block_offset)
{
/* if all string indexes are unique, highest_str_index equals
total_ref_count. otherwise it's always lower. */
"Corrupted strmap index file %s: "
"String indexes too high "
"(highest=%u max=%u)",
return -1;
}
if (ctx->lost_expunged_uid != 0) {
/* our view contained a message that had since been expunged. */
} else if (ret < 0) {
else
return -1;
}
return ret;
}
static bool
const struct mail_index_strmap_rec *hash_rec,
struct hash2_iter *iter)
{
/* hopefully it's a message that has since been expunged */
/* message is no longer in our view. remove it completely. */
return TRUE;
}
/* it's quite likely a conflict. we may not be able to verify
it, so just assume it is. nothing breaks even if we guess
wrong, the performance just suffers a bit. */
return FALSE;
}
/* 0 means "doesn't match", which is the only acceptable case */
}
static int
{
struct mail_index_strmap_rec *hash_rec;
struct hash2_iter iter;
if (crc32 == 0) {
/* unique string - there are no conflicts */
return 0;
}
/* check for conflicting string indexes. they may happen if
1) msgid exists only for a message X that has been expunged
2) another process doesn't see X, but sees msgid for another
message and writes it using a new string index
3) if we still see X, we now see the same msgid with two
string indexes.
if we detect such a conflict, we can't continue using the
strmap index until X has been expunged. */
/* CRC32 matches, but string index doesn't */
return -1;
}
}
return 0;
}
static int
{
struct mail_index_strmap_rec *hash_rec;
int ret;
/* we've already added this */
continue;
}
}
ret = -1;
break;
}
/* add the record to records array */
/* add a separate copy of the record to hash */
}
}
struct mail_index_strmap_view_sync *
{
struct mail_index_strmap_view_sync *sync;
struct mail_index_strmap_read_context ctx;
int ret;
if (mail_index_strmap_refresh(view) < 0) {
/* reading the strmap failed - just ignore and do
this in-memory based on whatever we knew last */
if (mail_index_strmap_view_sync_block(&ctx) < 0) {
ret = -1;
break;
}
if (ctx.too_large_uids)
break;
}
if (ret < 0) {
/* something failed - we can still use the strmap as far
as we managed to read it, but our view is now out
of sync */
} else {
}
}
return sync;
}
{
}
const char *key)
{
struct mail_index_strmap_hash_key hash_key;
/* The string already exists, use the same unique idx */
} else {
/* Newly seen string, assign a new unique idx to it */
}
}
{
struct mail_index_strmap_rec rec;
}
static void
{
/* zero-terminate the records array */
}
{
struct mail_index_strmap_read_context ctx;
int ret;
/* create a map of old -> new index and remove records of
expunged messages */
/* see if this record should be removed */
if (ret == 0) {
/* message expunged */
do {
i++;
continue;
}
}
if (i != dest) {
}
i++; dest++;
}
i_assert(renumber_map[0] == 0);
/* notify caller of the renumbering */
view->cb_context);
/* renumber the indexes in-place and recreate the hash */
for (i = 0; i < count; i++) {
}
/* update the new next_str_idx only after remapping */
}
{
const struct mail_index_strmap_rec *recs;
/* skip over the block size for now, we don't know it yet */
block_size = 0;
/* write records */
while (i < count) {
/* @UNSAFE: <uid diff> */
p = packed;
/* find how many records belong to this UID */
uid_rec_count = 1;
for (j = i + 1; j < count; j++) {
break;
}
/* <n> <crc32>*count <str_idx>*count -
FIXME: thread index specific code */
if (uid_rec_count == 1) {
/* Only Message-ID: header */
n = 0;
/* In-Reply-To: header */
n = 1;
} else {
/* References: header */
n = uid_rec_count;
}
mail_index_pack_num(&p, n);
for (j = 0; j < uid_rec_count; j++)
for (j = 0; j < uid_rec_count; j++) {
}
i += uid_rec_count;
}
/* we know the block size now - write it */
i_assert(block_size != 0);
if (output->last_failed_errno != 0)
return -1;
else {
return 0;
}
}
static void
{
const struct mail_index_header *idx_hdr;
struct mail_index_strmap_header hdr;
/* write header */
view->total_ref_count = 0;
}
{
const char *temp_path;
/* everything expunged - just unlink the existing index */
return 0;
}
if (fd == -1) {
"safe_mkstemp_hostpid(%s) failed: %m",
return -1;
}
if (output->last_failed_errno != 0) {
"write(%s) failed: %m", temp_path);
ret = -1;
}
"close(%s) failed: %m", temp_path);
ret = -1;
"rename(%s, %s) failed: %m",
ret = -1;
}
if (ret < 0)
return ret;
}
{
int ret;
if (ret <= 0) {
"file_wait_lock()");
}
} else {
if (ret <= 0) {
"file_dotlock_create()");
}
}
return ret;
}
{
}
{
}
static int
{
struct mail_index_strmap_read_context ctx;
const struct mail_index_strmap_rec *old_recs;
unsigned int i, old_count;
bool full_block;
int ret;
/* Check first if another process had written new records to the file.
If there are any, hopefully they're the same as what we would be
writing. There are two problematic cases when messages have been
expunged recently:
1) The file contains UIDs that we don't have. This means the string
indexes won't be compatible anymore, so we'll have to renumber ours
to match the ones in the strmap file.
Currently we don't bother handling 1) case. If indexes don't match
what we have, we just don't write anything.
2) We have UIDs that don't exist in the file. We can't simply skip
those records, because other records may have pointers to them using
different string indexes than we have. Even if we renumbered those,
future appends by other processes might cause the same problem (they
see the string for the first time and assign it a new index, but we
already have internally given it another index). So the only
sensible choice is to write nothing and hope that the message goes
away soon. */
sizeof(*old_recs), strmap_rec_cmp, &i);
if (i < old_count) {
i--;
}
while (i < old_count &&
/* mismatch */
/* 1) case */
/* 2) case */
} else {
/* string index mismatch,
shouldn't happen */
}
ret = -1;
break;
}
if (++i == old_count) {
full_block = FALSE;
break;
}
}
ret = -1;
break;
}
}
if (ret < 0)
return -1;
if (i == old_count) {
/* nothing new to write */
return 0;
}
/* write the new records */
ret = -1;
}
return ret;
}
{
int ret;
/* FIXME: this renumbering doesn't work well when running for a long
time since records aren't removed from hash often enough */
if (mail_index_strmap_recreate(view) < 0) {
return -1;
}
}
return 0;
}
return 0;
/* initial file creation */
if (mail_index_strmap_recreate(view) < 0) {
return -1;
}
return 0;
}
/* append the new records to the strmap file */
/* timeout / error */
ret = -1;
/* the file was already recreated - leave the syncing as it is
for now and let the next sync re-read the file. */
ret = 0;
} else {
}
if (ret < 0)
return ret;
}
{
(void)mail_index_strmap_write(view);
/* zero-terminate the records array */
}
{
}