dbox-map.c revision bbef9a6fc46440df829571756a7b75a95aba9d36
/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "hash.h"
#include "ostream.h"
#include "mkdir-parents.h"
#include "dbox-storage.h"
#include "dbox-file.h"
#include "dbox-map-private.h"
#define MAX_BACKWARDS_LOOKUPS 10
#define DBOX_FORCE_PURGE_MIN_RATIO 0.5
struct dbox_map_transaction_context {
struct mail_index_transaction *trans;
struct mail_index_sync_ctx *sync_ctx;
unsigned int changed:1;
unsigned int success:1;
};
{
"dbox map %s corrupted: %s",
}
{
sizeof(struct dbox_mail_index_map_header),
sizeof(struct dbox_mail_index_map_record),
sizeof(uint32_t));
return map;
}
{
}
{
storage->create_gid_origin) < 0 &&
return -1;
}
return 0;
}
{
int ret;
/* already opened */
return 0;
}
if (create_missing) {
return -1;
}
if (ret < 0) {
return -1;
}
if (ret == 0) {
/* index not found - for now just return failure */
return -1;
}
return 0;
}
{
struct mail_index_view_sync_ctx *ctx;
bool delayed_expunges;
/* some open files may have read partially written mails. now that
map syncing makes the new mails visible, we need to make sure the
partial data is flushed out of memory */
return -1;
}
return -1;
}
return 0;
}
{
const struct dbox_mail_index_map_record *rec;
const void *data;
bool expunged;
return -1;
}
return 0;
}
static int
{
/* not found - try again after a refresh */
if (dbox_map_refresh(map) < 0)
return -1;
return 0;
}
return 1;
}
{
int ret;
return -1;
return ret;
return -1;
return 1;
}
{
const void *data;
bool expunged;
return -1;
}
return -1;
}
return 0;
}
{
const struct mail_index_header *hdr;
struct dbox_mail_lookup_rec rec;
struct dbox_map_file_msg msg;
if (dbox_map_refresh(map) < 0)
return -1;
return -1;
}
}
return 0;
}
struct dbox_file_size {
};
{
struct hash_table *hash;
struct dbox_file_size *size;
struct seq_range_iter iter;
const struct mail_index_header *hdr;
const struct dbox_mail_index_map_record *rec;
const void *data;
unsigned int i;
bool expunged;
/* count file sizes */
continue;
continue;
/* this file has at least some zero references. count how many
bytes it has in total and how much of it has refcount=0. */
continue;
size);
}
if (*ref16_p == 0)
}
/* now drop the files that don't have enough deleted space */
}
pool_unref(&pool);
}
{
const struct mail_index_header *hdr;
const struct dbox_mail_index_map_record *rec;
const void *data;
bool expunged;
/* we never purge anything */
return FALSE;
}
ref0_size = total_size = 0;
continue;
continue;
if (*ref16_p == 0)
}
return FALSE;
return FALSE;
return TRUE;
}
{
const struct mail_index_header *hdr;
const struct dbox_mail_index_map_record *rec;
const void *data;
bool expunged;
else
/* we're never purging anything */
return &map->ref0_file_ids;
}
/* some internal error */
return &map->ref0_file_ids;
}
(void)dbox_map_refresh(map);
if (*ref16_p != 0)
continue;
}
}
}
return &map->ref0_file_ids;
}
struct dbox_map_transaction_context *
{
struct dbox_map_transaction_context *ctx;
if (external)
dbox_map_refresh(map) == 0)
return ctx;
}
static void
{
struct mail_index_sync_rec sync_rec;
/* something had crashed. need a full resync. */
i_warning("dbox %s: Inconsistency in map index "
} else {
}
}
{
struct mail_index_view *view;
struct mail_index_transaction *sync_trans;
int ret;
return 0;
/* use syncing to lock the transaction log, so that we always see
log's head_offset = tail_offset */
&view, &sync_trans, 0);
if (ret <= 0) {
return -1;
}
return -1;
}
return 0;
}
{
}
}
}
{
unsigned int i, count;
const void *data;
bool expunged;
int cur_diff;
return -1;
for (i = 0; i < count; i++) {
/* we can't refresh map here since view has a
transaction open. */
"refcount update lost map_uid=%u", *uidp);
return -1;
}
if (cur_diff >= 32768) {
/* we're getting close to the 64k limit. fail early
to make it less likely that two processes increase
the refcount enough times to cross the limit */
"Message has been copied too many times");
return -1;
}
}
return 0;
}
{
struct dbox_map_transaction_context *map_trans;
const struct mail_index_header *hdr;
const struct dbox_mail_index_map_record *rec;
const void *data;
bool expunged;
int ret = 0;
/* make sure the map is refreshed, otherwise we might be expunging
messages that have already been moved to other files. */
/* we need a per-file transaction, otherwise we can't refresh the map */
ret = -1;
break;
}
}
}
if (ret == 0)
(void)dbox_map_transaction_commit(map_trans);
return ret;
}
struct dbox_map_append_context *
{
struct dbox_map_append_context *ctx;
else {
/* refresh the map so we can try appending to the
latest files */
}
return ctx;
}
struct dbox_map_append_context *
{
struct dbox_map_append_context *ctx;
return ctx;
}
{
if (days == 0)
return 0;
/* get beginning of today */
i_panic("mktime(today) failed");
}
static bool
bool *retry_later_r)
{
int ret;
*retry_later_r = FALSE;
return TRUE;
}
/* already locked, we're possibly in the middle of purging it
in which case we really don't want to write there. */
return TRUE;
}
file_too_old = TRUE;
/* locking failed */
*retry_later_r = ret == 0;
/* the file was unlinked between opening and locking it. */
output_r) <= 0) {
/* couldn't append to this file */
/* file was too large after all */
} else {
/* success */
return TRUE;
}
/* failure */
return !file_too_old;
}
static bool
{
unsigned int i, count;
/* there shouldn't be many files open, don't bother with anything
faster. */
for (i = 0; i < count; i++) {
return TRUE;
}
return FALSE;
}
static int
{
const struct mail_index_header *hdr;
unsigned int i, count, backwards_lookup_count;
bool retry_later;
*existing_r = FALSE;
return 0;
/* first try to use files already used in this append */
/* we already decided we can't append to this */
continue;
}
output_r) > 0) {
*existing_r = TRUE;
return 1;
}
/* can't append to this file anymore */
#if 0 /* FIXME: we can't close files, otherwise we lose the lock too early */
/* avoid wasting fds by closing the file, but not if
we're also reading from it. */
return -1;
}
#endif
}
/* try to find an existing appendable file */
return -1;
continue;
if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
/* we've wasted enough time here */
break;
}
/* first lookup: this should be enough usually, but we can't
be sure until after locking. also if messages were recently
moved, this message might not be the last one in the file. */
continue;
/* already checked this */
continue;
}
/* file is too old. the rest of the files are too. */
break;
}
/* NOTE: we've now refreshed map view. there are no guarantees
about sequences anymore. */
return 1;
/* FIXME: use retry_later somehow */
if (uid == 1 ||
break;
seq++;
}
return 0;
}
{
struct dbox_map_append *append;
bool existing;
int ret;
return -1;
if (ret < 0)
return -1;
if (ret == 0) {
/* create a new file */
output_r);
if (ret <= 0) {
return -1;
}
}
}
if (!existing) {
}
return 0;
}
{
struct dbox_map_append *appends;
unsigned int count;
}
static int
{
const struct dbox_mail_index_map_header *hdr;
const void *data;
if (data_size != 0) {
return -1;
}
/* first file */
*file_id_r = 1;
} else {
}
return 0;
}
bool separate_transaction)
{
unsigned int i, count;
int ret;
/* start the syncing. we'll need it even if there are no file ids to
be assigned. */
if (ret <= 0) {
return -1;
}
return -1;
}
/* assign file_ids for newly created multi-files */
for (i = 0; i < count; i++) {
continue;
if (dbox_file_flush_append(files[i]) < 0) {
ret = -1;
break;
}
}
ret = -1;
break;
}
}
}
if (ret < 0) {
return -1;
}
/* update the highest used file_id */
if (first_file_id != file_id) {
file_id--;
}
return 0;
}
{
const struct dbox_map_append *appends;
const struct mail_index_header *hdr;
struct dbox_mail_index_map_record rec;
unsigned int i, count;
int ret = 0;
*first_map_uid_r = 0;
*last_map_uid_r = 0;
return 0;
}
return -1;
/* append map records to index */
ref16 = 1;
for (i = 0; i < count; i++) {
}
/* assign map UIDs for appended records */
if (hdr->uid_validity == 0) {
/* we don't really care about uidvalidity, but it can't be 0 */
}
return -1;
}
return ret;
}
{
const struct dbox_map_append *appends;
struct dbox_mail_index_map_record rec;
struct seq_range_iter iter;
unsigned int i, j, map_uids_count, appends_count;
return -1;
for (i = j = 0; i < map_uids_count; i++) {
i_assert(j < appends_count);
j++;
i_unreached();
}
i_unreached();
}
return 0;
}
{
struct seq_range_iter iter;
unsigned int i, count, n = 0;
bool ret;
for (i = 0; i < count; i++) {
continue;
return -1;
}
return 0;
}
{
return -1;
}
}
return 0;
}
{
/* flush before truncating */
}
}
} else {
}
}
}
{
unsigned int i, count;
for (i = 0; i < count; i++) {
files[i]->first_append_offset = 0;
dbox_file_unlock(files[i]);
dbox_file_unref(&files[i]);
}
}
{
const struct mail_index_header *hdr;
if (hdr->uid_validity != 0)
return hdr->uid_validity;
/* refresh index in case it was just changed */
(void)dbox_map_refresh(map);
}