mdbox-map.c revision 18122b1fb9aa58945ed12d7020e91fbf4f04a0cc
/* Copyright (c) 2007-2017 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "hash.h"
#include "ostream.h"
#include "mkdir-parents.h"
#include "unlink-old-files.h"
#include "mailbox-list-private.h"
#include "mdbox-storage.h"
#include "mdbox-file.h"
#include "mdbox-map-private.h"
#include <dirent.h>
#define MAX_BACKWARDS_LOOKUPS 10
#define DBOX_FORCE_PURGE_MIN_RATIO 0.5
struct mdbox_map_transaction_context {
struct mdbox_map_atomic_context *atomic;
struct mail_index_transaction *trans;
bool changed:1;
bool committed:1;
};
{
"mdbox map %s corrupted: %s",
}
struct mdbox_map *
{
const char *root, *index_root;
map->index_path =
sizeof(struct mdbox_map_mail_index_header),
sizeof(struct mdbox_map_mail_index_record),
sizeof(uint32_t));
return map;
}
{
}
}
{
MAILBOX_LIST_PATH_TYPE_DIR) < 0) {
return -1;
}
MAILBOX_LIST_PATH_TYPE_INDEX) < 0) {
return -1;
}
return 0;
}
{
unsigned int interval =
return;
/* check once in a while if there are temp files to clean up */
if (interval == 0) {
/* disabled */
/* there haven't been any changes to this directory since we
last checked it. */
/* time to scan */
}
}
{
struct mailbox_permissions perm;
int ret = 0;
/* already opened */
return 1;
}
if (create_missing) {
return -1;
if (ret > 0) {
/* storage/ directory already existed.
the index should exist also. */
} else {
}
}
if (ret == 0 && create_missing) {
/* storage/ already existed, but indexes didn't. we'll need to
take extra steps to make sure we won't overwrite any m.*
files that may already exist. */
}
if (ret < 0) {
return -1;
}
if (ret == 0) {
/* index not found - for now just return failure */
return 0;
}
if (mdbox_map_generate_uid_validity(map) < 0) {
return -1;
}
if (mdbox_map_refresh(map) < 0) {
return -1;
}
}
return 1;
}
{
}
{
}
{
struct mail_index_view_sync_ctx *ctx;
bool delayed_expunges, fscked;
int ret = 0;
/* some open files may have read partially written mails. now that
map syncing makes the new mails visible, we need to make sure the
partial data is flushed out of memory */
return -1;
}
/* can't sync when there are transactions */
return 0;
}
ret = -1;
}
if (fscked)
return ret;
}
{
const struct mail_index_header *hdr;
/* map isn't opened yet. don't bother. */
return FALSE;
}
}
static void
struct mdbox_map_mail_index_header *hdr_r)
{
const void *data;
}
{
struct mdbox_map_mail_index_header hdr;
return hdr.rebuild_count;
}
static int
const struct mdbox_map_mail_index_record **rec_r)
{
const struct mdbox_map_mail_index_record *rec;
const void *data;
return -1;
}
return 0;
}
static int
{
/* not found - try again after a refresh */
if (mdbox_map_refresh(map) < 0)
return -1;
return 0;
}
return 1;
}
{
const struct mdbox_map_mail_index_record *rec;
int ret;
if (mdbox_map_open_or_create(map) < 0)
return -1;
return ret;
return -1;
return 1;
}
struct mdbox_map_mail_index_record *rec_r,
{
int ret;
if (mdbox_map_open_or_create(map) < 0)
return -1;
return ret;
}
struct mdbox_map_mail_index_record *rec_r,
{
const struct mdbox_map_mail_index_record *rec;
const void *data;
return -1;
return -1;
}
*refcount_r = *ref16_p;
return 1;
}
{
return uid;
}
{
}
struct dbox_mail_lookup_rec *rec_r)
{
const void *data;
return -1;
}
return -1;
}
return 0;
}
{
const struct mail_index_header *hdr;
struct dbox_mail_lookup_rec rec;
struct mdbox_map_file_msg msg;
if (mdbox_map_refresh(map) < 0)
return -1;
return -1;
}
}
return 0;
}
{
const struct mail_index_header *hdr;
const struct mdbox_map_mail_index_record *rec;
const void *data;
bool expunged;
int ret;
/* no map / internal error */
return ret;
}
if (mdbox_map_refresh(map) < 0)
return -1;
if (*ref16_p != 0)
continue;
}
}
}
return 0;
}
{
struct mdbox_map_atomic_context *atomic;
return atomic;
}
static void
struct mail_index_sync_ctx *sync_ctx)
{
struct mail_index_sync_rec sync_rec;
/* something had crashed. need a full resync. */
i_warning("mdbox %s: Inconsistency in map index "
}
}
const char *reason)
{
int ret;
return 0;
return -1;
/* use syncing to lock the transaction log, so that we always see
log's head_offset = tail_offset */
if (ret <= 0) {
return -1;
}
/* reset refresh state so that if it's wanted to be done locked,
it gets the latest changes */
return 0;
}
{
}
{
}
{
}
{
}
{
int ret = 0;
/* not locked */
ret = -1;
}
} else {
}
return ret;
}
struct mdbox_map_transaction_context *
bool external)
{
struct mdbox_map_transaction_context *ctx;
bool success;
if (external)
/* already refreshed within a lock, don't do it again */
} else {
}
if (success) {
flags);
}
return ctx;
}
const char *reason)
{
return 0;
return -1;
return -1;
}
return 0;
}
{
}
{
const void *data;
return -1;
/* we can't refresh map here since view has a
transaction open. */
if (diff > 0) {
/* the message was probably just purged */
"Some of the requested messages no longer exist.");
} else {
"refcount update lost map_uid=%u", map_uid);
}
return -1;
}
map_uid);
return -1;
}
/* we're getting close to the 64k limit. fail early
to make it less likely that two processes increase
the refcount enough times to cross the limit */
t_strdup_printf("Message has been copied too many times (%d + %d)",
return -1;
}
return 0;
}
{
unsigned int i, count;
return -1;
for (i = 0; i < count; i++) {
return -1;
}
return 0;
}
{
struct mdbox_map_atomic_context *atomic;
struct mdbox_map_transaction_context *map_trans;
const struct mail_index_header *hdr;
const struct mdbox_map_mail_index_record *rec;
const void *data;
int ret = 0;
/* make sure the map is refreshed, otherwise we might be expunging
messages that have already been moved to other files. */
/* we need a per-file transaction, otherwise we can't refresh the map */
ret = -1;
break;
}
}
}
if (ret == 0)
if (mdbox_map_atomic_finish(&atomic) < 0)
ret = -1;
return ret;
}
struct mdbox_map_append_context *
{
struct mdbox_map_append_context *ctx;
else {
/* refresh the map so we can try appending to the
latest files */
else
}
return ctx;
}
{
unsigned int unit = 1;
if (interval == 0)
return 0;
the interval is */
if (interval >= 60) {
unit = 60;
if (interval >= 3600) {
unit = 3600;
}
}
}
i_panic("mktime(today) failed");
}
{
bool notfound;
if (want_altpath) {
return FALSE;
} else {
return FALSE;
}
if (notfound)
return FALSE;
/* already locked, we're possibly in the middle of purging it
in which case we really don't want to write there. */
return FALSE;
}
/* different alt location than what we want, can't use it */
return FALSE;
}
return TRUE;
}
{
bool last;
int ret;
return FALSE;
return FALSE;
return TRUE;
}
static bool
bool want_altpath,
const struct mdbox_map_mail_index_record *rec,
struct dbox_file_append_context **file_append_r,
{
struct dbox_file_append_context *file_append;
bool file_too_old = FALSE;
int ret;
*file_append_r = NULL;
*retry_later_r = FALSE;
return TRUE;
}
file_too_old = TRUE;
/* locking failed */
*retry_later_r = ret == 0;
/* the file was unlinked between opening and locking it. */
/* check if there's any garbage at the end of file.
note that there may be valid messages added by another
session before we locked it (but after we refreshed
map index). */
/* error message was already logged */
} else {
/* couldn't append to this file */
/* file was too large after all */
} else {
/* success */
return TRUE;
}
}
/* failure */
return !file_too_old;
}
static bool
{
struct dbox_file_append_context *const *file_appends;
unsigned int i, count;
/* there shouldn't be many files open, don't bother with anything
faster. */
for (i = 0; i < count; i++) {
struct mdbox_file *mfile =
return TRUE;
}
return FALSE;
}
static struct dbox_file_append_context *
{
struct mdbox_file *mfile;
unsigned int i, count;
/* first try to use files already used in this append */
continue;
/* already closed it (below). we might be able to still
fit some small mail there, but that's too much
trouble */
continue;
}
return append;
/* can't append to this file anymore. if we created this file,
close it so we don't waste fds. if we didn't, we can't close
it without also losing our lock too early. */
}
return NULL;
}
static int
{
struct dirent *d;
int ret = 0;
/* we want to quickly find the latest alt file, but we also want to
avoid accessing the alt storage as much as possible. typically most
of the older mails would be in alt storage, so we'll just put the
few m.* files in primary storage to checked_file_ids array. other
files are then known to exist in alt storage. */
return -1;
}
strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
continue;
&file_id) < 0)
continue;
}
if (errno != 0) {
ret = -1;
}
ret = -1;
}
return ret;
}
static int
struct dbox_file_append_context **file_append_r,
{
const struct mail_index_header *hdr;
const struct mdbox_map_mail_index_record *rec;
unsigned int backwards_lookup_count;
bool retry_later;
return 0;
/* try to find an existing appendable file */
if (want_altpath) {
/* we want to save to alt storage. */
return -1;
}
return -1;
continue;
if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
/* we've wasted enough time here */
break;
}
/* first lookup: this should be enough usually, but we can't
be sure until after locking. also if messages were recently
moved, this message might not be the last one in the file. */
continue;
/* already checked this */
continue;
}
output_r, &retry_later)) {
/* file is too old. the rest of the files are too. */
break;
}
/* NOTE: we've now refreshed map view. there are no guarantees
about sequences anymore. */
if (*file_append_r != NULL)
return 1;
/* FIXME: use retry_later somehow */
if (uid == 1 ||
break;
seq++;
}
return 0;
}
struct dbox_file_append_context **file_append_ctx_r,
{
struct mdbox_map_append *append;
struct dbox_file_append_context *file_append;
bool existing, want_altpath;
int ret;
return -1;
if (file_append != NULL) {
ret = 1;
} else {
&file_append, output_r);
}
if (ret > 0)
else if (ret < 0)
return -1;
else {
/* create a new file */
if (ret <= 0) {
return -1;
}
}
if (!existing) {
}
return 0;
}
static void
struct dbox_file_append_context *append_ctx)
{
struct mdbox_file *mfile =
/* if this file is now large enough not to fit any other
mails and we created it, close its fd since it's not
needed anymore. */
dbox_file_append_flush(append_ctx) == 0)
}
{
unsigned int count;
}
{
struct mdbox_map_append *appends;
unsigned int count;
}
static int
{
struct dirent *d;
unsigned int id, highest_id = 0;
return -1;
}
if (highest_id < id)
highest_id = id;
}
}
*file_id_r = highest_id;
return 0;
}
static int
bool separate_transaction, const char *reason)
{
struct dbox_file_append_context *const *file_appends;
unsigned int i, count;
struct mdbox_map_mail_index_header hdr;
/* start the syncing. we'll need it even if there are no file ids to
be assigned. */
return -1;
/* storage/ directory had been already created but
without indexes. scan to see if there exists a higher
m.* file id than what is in header, so we won't
accidentally overwrite any existing files. */
return -1;
}
/* assign file_ids for newly created files */
for (i = 0; i < count; i++) {
struct mdbox_file *mfile =
if (dbox_file_append_flush(file_appends[i]) < 0)
return -1;
return -1;
}
}
/* update the highest used file_id */
if (first_file_id != file_id) {
file_id--;
}
return 0;
}
{
const struct mdbox_map_append *appends;
const struct mail_index_header *hdr;
struct mdbox_map_mail_index_record rec;
unsigned int i, count;
int ret = 0;
*first_map_uid_r = 0;
*last_map_uid_r = 0;
return 0;
}
return -1;
/* append map records to index */
ref16 = 1;
for (i = 0; i < count; i++) {
struct mdbox_file *mfile =
}
/* assign map UIDs for appended records */
if (hdr->uid_validity == 0) {
/* we don't really care about uidvalidity, but it can't be 0 */
}
return -1;
}
return ret;
}
{
const struct mdbox_map_append *appends;
struct mdbox_map_mail_index_record rec;
struct seq_range_iter iter;
unsigned int i, j, map_uids_count, appends_count;
/* map is locked by this call */
return -1;
for (i = j = 0; i < map_uids_count; i++) {
struct mdbox_file *mfile =
i_assert(j < appends_count);
j++;
/* We wrote the email to the new m.* file, but another
process already expunged it and purged it. Deleting
the email from the new m.* file would be problematic
at this point, so just add the mail back to the map
with refcount=0 and the next purge will remove it. */
}
}
i_unreached();
}
return 0;
}
{
struct dbox_file_append_context **file_appends;
unsigned int i, count;
for (i = 0; i < count; i++) {
if (dbox_file_append_flush(file_appends[i]) < 0)
return -1;
}
return 0;
}
{
struct dbox_file_append_context **file_appends;
unsigned int i, count;
for (i = 0; i < count; i++) {
if (dbox_file_append_commit(&file_appends[i]) < 0)
return -1;
}
return 0;
}
{
struct dbox_file_append_context **file_appends;
unsigned int i, count;
for (i = 0; i < count; i++) {
if (file_appends[i] != NULL)
}
for (i = 0; i < count; i++) {
dbox_file_unlock(files[i]);
dbox_file_unref(&files[i]);
}
}
{
const struct mail_index_header *hdr;
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_view *view;
struct mail_index_transaction *trans;
int ret;
/* do this inside syncing, so that we're locked and there are no
race conditions */
if (ret <= 0) {
return -1;
}
if (hdr->uid_validity != 0) {
/* someone else beat us to it */
} else {
}
return mail_index_sync_commit(&sync_ctx);
}
{
if (uid_validity == 0)
return uid_validity;
}