/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "bsearch-insert-pos.h"
#include "istream.h"
#include "ostream.h"
#include "file-lock.h"
#include "file-dotlock.h"
#include "crc32.h"
#include "safe-mkstemp.h"
#include "str.h"
#include "mail-index-private.h"
#include "mail-index-strmap.h"
#include <stdio.h>
struct mail_index_strmap {
char *path;
int fd;
};
struct mail_index_strmap_view {
void *cb_context;
};
struct mail_index_strmap_read_context {
unsigned int rec_size;
};
struct mail_index_strmap_view_sync {
};
struct mail_index_strmap_hash_key {
const char *str;
};
/* number of bytes required to store one string idx */
/* renumber the string indexes when highest string idx becomes larger than
<number of indexes>*STRMAP_FILE_MAX_STRIDX_MULTIPLIER */
.stale_timeout = 30
};
struct mail_index_strmap *
{
return strmap;
}
static bool
static void
const char *function)
{
return;
}
"%s failed with strmap index file %s: %m",
}
{
}
}
{
}
{
}
static bool
{
}
struct mail_index_strmap_view *
struct mail_index_view *idx_view,
void *context,
const struct hash2_table **hash_r)
{
return view;
}
{
}
{
}
{
view->last_added_uid = 0;
view->lost_expunged_uid = 0;
}
{
"Corrupted strmap index file: %s",
}
{
const unsigned char *data;
int ret;
return 0;
return -1;
}
if (ret <= 0) {
if (ret < 0) {
} else {
}
return ret;
}
/* need to rebuild. if we already had something in the strmap,
we can keep it. */
return 0;
}
/* we'll read the entire file from the beginning */
view->last_added_uid = 0;
view->last_read_uid = 0;
view->total_ref_count = 0;
return 0;
}
{
/* FIXME: nfs flush */
if (!ESTALE_FSTAT(errno))
return TRUE;
}
return TRUE;
}
}
{
return -1;
if (view->lost_expunged_uid != 0) {
/* last read failed because view had a message
that didn't exist in the strmap (because it
was expunged by another session). if the
message still isn't expunged in this view,
just continue using the current strmap. */
return -1;
/* our view isn't synced with the disk, we
can't read strmap without first resetting
the view */
} else {
return 0;
}
}
}
return mail_index_strmap_open(view);
}
static int
{
const unsigned char *data;
int ret;
if (ret <= 0)
return ret;
return -1;
return 1;
}
static int
{
/* thread index has larger UIDs than what we've seen
in our view. we'll have to read them again later
when we know about them */
}
return 0;
}
ctx->uid_lookup_seq++;
return 1;
return 0;
} else {
/* record that exists in index is missing from strmap.
see if it's because the strmap is corrupted or because
our current view is a bit stale and the message has already
been expunged. */
return -1;
}
}
static int
{
int ret;
/* <uid> <n> <crc32>*count <str_idx>*count
where
n = 0 -> count=1 (only Message-ID:)
n = 1 -> count=2 (Message-ID: + In-Reply-To:)
n = 2+ -> count=n (Message-ID: + References:)
*/
if (mail_index_strmap_read_packed(ctx, &n) <= 0)
return -1;
if (ret < 0)
return -1;
return -1;
if (ret == 0) {
/* this message has already been expunged, ignore it.
update highest string indexes anyway. */
for (i = 0; i < count; i++) {
}
return 0;
}
/* everything exists. save it. FIXME: these ref_index values
are thread index specific, perhaps something more generic
should be used some day */
ctx->next_ref_index = 0;
i_unreached();
return 1;
}
static bool
{
return FALSE;
}
/* FIXME: str_idx could be stored as packed relative values
(first relative to highest_idx, the rest relative to the
previous str_idx) */
/* read the record contents */
/* get to the next record */
return TRUE;
}
static int
struct mail_index_strmap_read_context *ctx)
{
const unsigned char *data;
int ret;
/* come back later when we know about the new UIDs */
return 0;
}
sizeof(block_size));
if (ret <= 0) {
/* no new data */
return 0;
}
return -1;
}
if (block_size == 0) {
/* the rest of the file is either not written, or the previous
write didn't finish */
return 0;
}
/* block size too large */
return -1;
}
/* FIXME: when reading multiple blocks we shouldn't have to calculate
this every time */
return 1;
}
static int
{
int ret;
return 1;
/* get next UID */
do {
/* this block is done */
return 0;
}
return -1;
} while (ret == 0);
return ret;
}
static int
bool update_block_offset)
{
/* if all string indexes are unique, highest_str_index equals
total_ref_count. otherwise it's always lower. */
"Corrupted strmap index file %s: "
"String indexes too high "
"(highest=%u max=%u)",
return -1;
}
if (ctx->lost_expunged_uid != 0) {
/* our view contained a message that had since been expunged. */
} else if (ret < 0) {
else
return -1;
}
return ret;
}
static bool
const struct mail_index_strmap_rec *hash_rec,
struct hash2_iter *iter)
{
/* hopefully it's a message that has since been expunged */
/* message is no longer in our view. remove it completely. */
return TRUE;
}
/* it's quite likely a conflict. we may not be able to verify
it, so just assume it is. nothing breaks even if we guess
wrong, the performance just suffers a bit. */
return FALSE;
}
/* 0 means "doesn't match", which is the only acceptable case */
}
static int
{
if (crc32 == 0) {
/* unique string - there are no conflicts */
return 0;
}
/* check for conflicting string indexes. they may happen if
1) msgid exists only for a message X that has been expunged
2) another process doesn't see X, but sees msgid for another
message and writes it using a new string index
3) if we still see X, we now see the same msgid with two
string indexes.
if we detect such a conflict, we can't continue using the
strmap index until X has been expunged. */
/* CRC32 matches, but string index doesn't */
return -1;
}
}
return 0;
}
static int
{
int ret;
/* we've already added this */
continue;
}
}
ret = -1;
break;
}
/* add the record to records array */
/* add a separate copy of the record to hash */
}
}
struct mail_index_strmap_view_sync *
{
int ret;
if (mail_index_strmap_refresh(view) < 0) {
/* reading the strmap failed - just ignore and do
this in-memory based on whatever we knew last */
if (mail_index_strmap_view_sync_block(&ctx) < 0) {
ret = -1;
break;
}
if (ctx.too_large_uids)
break;
}
if (ret < 0) {
/* something failed - we can still use the strmap as far
as we managed to read it, but our view is now out
of sync */
} else {
}
}
return sync;
}
{
/* we'll flip the bits because of a bug in our old crc32 code.
this keeps the index format backwards compatible with the new fixed
crc32 code. */
}
const char *key)
{
/* The string already exists, use the same unique idx */
} else {
/* Newly seen string, assign a new unique idx to it */
}
}
{
}
static void
{
/* zero-terminate the records array */
}
{
int ret;
/* create a map of old -> new index and remove records of
expunged messages */
/* see if this record should be removed */
if (ret == 0) {
/* message expunged */
do {
i++;
continue;
}
}
if (i != dest) {
}
i++; dest++;
}
i_assert(renumber_map[0] == 0);
/* notify caller of the renumbering */
view->cb_context);
/* renumber the indexes in-place and recreate the hash */
for (i = 0; i < count; i++) {
}
/* update the new next_str_idx only after remapping */
}
{
/* skip over the block size for now, we don't know it yet */
block_size = 0;
/* write records */
while (i < count) {
/* @UNSAFE: <uid diff> */
p = packed;
/* find how many records belong to this UID */
uid_rec_count = 1;
for (j = i + 1; j < count; j++) {
break;
}
/* <n> <crc32>*count <str_idx>*count -
FIXME: thread index specific code */
if (uid_rec_count == 1) {
/* Only Message-ID: header */
n = 0;
/* In-Reply-To: header */
n = 1;
} else {
/* References: header */
n = uid_rec_count;
}
mail_index_pack_num(&p, n);
for (j = 0; j < uid_rec_count; j++)
for (j = 0; j < uid_rec_count; j++) {
}
i += uid_rec_count;
}
/* we know the block size now - write it */
i_assert(block_size != 0);
if (output->stream_errno != 0)
return;
}
static void
{
/* write header */
view->total_ref_count = 0;
}
{
const char *temp_path;
/* everything expunged - just unlink the existing index */
return 0;
}
if (fd == -1) {
"safe_mkstemp_hostpid(%s) failed: %m",
return -1;
}
if (o_stream_finish(output) < 0) {
ret = -1;
}
"close(%s) failed: %m", temp_path);
ret = -1;
"rename(%s, %s) failed: %m",
ret = -1;
}
if (ret < 0)
return ret;
}
{
unsigned int timeout_secs;
int ret;
if (ret <= 0) {
"file_wait_lock()");
}
} else {
if (ret <= 0) {
"file_dotlock_create()");
}
}
return ret;
}
{
}
static int
{
}
static int
{
unsigned int i, old_count;
bool full_block;
int ret;
/* Check first if another process had written new records to the file.
If there are any, hopefully they're the same as what we would be
writing. There are two problematic cases when messages have been
expunged recently:
1) The file contains UIDs that we don't have. This means the string
indexes won't be compatible anymore, so we'll have to renumber ours
to match the ones in the strmap file.
Currently we don't bother handling 1) case. If indexes don't match
what we have, we just don't write anything.
2) We have UIDs that don't exist in the file. We can't simply skip
those records, because other records may have pointers to them using
different string indexes than we have. Even if we renumbered those,
future appends by other processes might cause the same problem (they
see the string for the first time and assign it a new index, but we
already have internally given it another index). So the only
sensible choice is to write nothing and hope that the message goes
away soon. */
strmap_rec_cmp, &i);
if (i < old_count) {
i--;
}
while (i < old_count &&
/* mismatch */
/* 1) case */
/* 2) case */
} else {
/* string index mismatch,
shouldn't happen */
}
ret = -1;
break;
}
if (++i == old_count) {
full_block = FALSE;
break;
}
}
ret = -1;
break;
}
}
if (ret < 0)
return -1;
if (i == old_count) {
/* nothing new to write */
return 0;
}
/* write the new records */
if (o_stream_finish(output) < 0) {
ret = -1;
}
return ret;
}
{
int ret;
/* FIXME: this renumbering doesn't work well when running for a long
time since records aren't removed from hash often enough */
if (mail_index_strmap_recreate(view) < 0) {
return -1;
}
}
return 0;
}
return 0;
/* initial file creation */
if (mail_index_strmap_recreate(view) < 0) {
return -1;
}
return 0;
}
/* append the new records to the strmap file */
/* timeout / error */
ret = -1;
/* the file was already recreated - leave the syncing as it is
for now and let the next sync re-read the file. */
ret = 0;
} else {
}
if (ret < 0)
return ret;
}
{
(void)mail_index_strmap_write(view);
/* zero-terminate the records array */
}
{
}