mdbox-purge.c revision 52de839a8249bff5eace53dc1401b28baa0c124b
/* Copyright (c) 2007-2013 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "istream.h"
#include "ostream.h"
#include "str.h"
#include "hash.h"
#include "dbox-attachment.h"
#include "mdbox-storage.h"
#include "mdbox-storage-rebuild.h"
#include "mdbox-file.h"
#include "mdbox-map.h"
#include "mdbox-sync.h"
#include <stdlib.h>
#include <dirent.h>
/*
Altmoving works like:
1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox
code and map UID's alt-refcount is updated. It won't be written to disk.
2. mdbox_purge() is called, which checks if map UID's refcount equals
to its alt-refcount. If it does, it's moved to alt storage. Moving to
primary storage is done if _ALT flag was removed from any message.
*/
enum mdbox_msg_action {
};
struct mdbox_purge_context {
struct mdbox_storage *storage;
/* list of file_ids that exist in primary storage. this list is looked
up while there is no locking, so it may not be accurate anymore by
the time it's used. */
/* list of file_ids that we need to purge */
/* uint32_t map_uid => enum mdbox_msg_action action */
HASH_TABLE(void *, void *) altmoves;
bool have_altmoves;
struct mdbox_map_atomic_context *atomic;
struct mdbox_map_append_context *append_ctx;
};
const struct mdbox_map_file_msg *m2)
{
return -1;
return 1;
else
return 0;
}
static int
struct dbox_metadata_header *meta_hdr_r)
{
const unsigned char *data;
int ret;
sizeof(*meta_hdr_r));
if (ret <= 0) {
return 0;
}
return -1;
}
sizeof(meta_hdr_r->magic_post)) != 0) {
return 0;
}
return 1;
}
static int
{
struct dbox_metadata_header meta_hdr;
const char *line;
int ret;
return ret;
/* use unlimited line length for metadata */
if (*line == '\0') {
/* end of metadata */
break;
}
}
return 0;
}
return 1;
}
static int
{
struct dbox_metadata_header meta_hdr;
const char *line;
int ret;
/* skip and ignore the header */
return ret;
/* use unlimited line length for metadata */
if (*line == '\0') {
/* end of metadata */
break;
}
extrefs)) {
i_warning("%s: Ignoring corrupted extref: %s",
}
} T_END;
}
return 0;
}
return 1;
}
static bool
{
enum mdbox_msg_action action;
void *value;
if (!ctx->have_altmoves)
return FALSE;
return action == MDBOX_MSG_ACTION_MOVE_TO_ALT;
}
static int
const struct mdbox_map_file_msg *msg)
{
struct dbox_file_append_context *out_file_append;
int read_errno;
return -1;
if (read_errno != 0) {
errno = read_errno;
return -1;
}
if (o_stream_nfinish(output) < 0) {
"write(%s) failed: %m",
return -1;
}
return 0;
}
/* copy metadata */
return ret;
return 1;
}
static int
{
struct mdbox_map_mail_index_record rec;
const struct mdbox_map_file_msg *msgs;
unsigned int i, count;
int ret;
return -1;
for (i = 0; i < count; i++) {
continue;
&refcount);
if (ret <= 0) {
if (ret < 0)
return -1;
"Purging unexpectedly lost map_uid=%u",
return -1;
}
if (refcount > 0)
return 0;
}
return 1;
}
static int
{
const struct mail_attachment_extref *extref;
int ret = 0;
ret = -1;
}
return ret;
}
static int
{
const struct mdbox_map_file_msg *msgs;
unsigned int i, count;
int ret;
return ret;
/* make sure the file still exists. another process may have already
deleted it. */
return 0;
return -1;
}
/* get list of map UIDs that exist in this file (again has to be done
after locking) */
&msgs_arr) < 0) {
return -1;
}
/* sort messages by their offset */
for (i = 0; i < count; i++) {
break;
/* map doesn't match file's actual contents */
"purging found mismatched offsets "
ret = 0;
break;
}
/* skip over expunged message */
/* skip metadata */
&ext_refs);
if (ret <= 0)
break;
} else {
/* non-expunged message. write it to output file. */
if (ret <= 0)
break;
}
}
/* file has more messages than what map tells us */
"more messages available than in map "
ret = 0;
}
if (ret <= 0)
ret = -1;
else {
/* it's possible that one of the messages we purged was
just copied to another mailbox. the only way to prevent that
would be to keep map locked during the purge, but that could
keep it locked for too long. instead we'll check here if
there are such copies, and if there are cancel this file's
purge. */
}
if (ret <= 0) {
/* failed */
/* everything purged from this file */
ret = 1;
} else {
/* assign new file_id + offset to moved messages */
&expunged_map_uids) < 0 ||
ret = -1;
else
ret = 1;
}
/* unlink only after unlocking map, so readers don't see it
temporarily vanished */
if (ret > 0) {
(void)dbox_file_unlink(file);
ret = -1;
} else {
}
return ret;
}
{
/* we'll assume here that alt flag won't be changed multiple times
for the same mail. it shouldn't happen with current code, and
checking for it would just slow down the code.
so the way it works currently is just that map_uids are added to
an array, which is later sorted and processed further. note that
it's still possible that the same map_uid exists in the array
multiple times. */
return;
if (!array_is_created(dest))
}
static struct mdbox_purge_context *
{
struct mdbox_purge_context *ctx;
return ctx;
}
{
}
{
struct dirent *d;
int ret = 0;
/* we don't need to do alt moving, don't bother getting list
of primary files */
return 0;
}
/* no storage directory at all yet */
return 0;
}
return -1;
}
strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
continue;
&file_id) < 0)
continue;
}
}
if (errno != 0) {
ret = -1;
}
ret = -1;
}
return ret;
}
{
return -1;
return 1;
return 0;
}
{
unsigned int i, count, alt_refcount = 0;
struct mdbox_map_mail_index_record cur_rec;
enum mdbox_msg_action action;
uint16_t cur_refcount = 0;
int ret = 0;
/* first add move-to-alt actions */
} else {
count = 0;
}
cur_map_uid = 0;
for (i = 0; i < count; i++) {
if (cur_map_uid != map_uids[i]) {
cur_map_uid = map_uids[i];
&cur_rec, &cur_refcount) < 0) {
ret = -1;
}
alt_refcount = 1;
} else {
alt_refcount++;
}
if (alt_refcount == cur_refcount &&
/* all instances marked as moved to alt storage */
}
}
/* next add move-from-alt actions. they override move-to-alt actions
in case there happen to be any conflicts (shouldn't). only a single
move-from-alt record is needed to do the move. */
else {
count = 0;
}
cur_map_uid = 0;
for (i = 0; i < count; i++) {
if (cur_map_uid == map_uids[i])
continue;
cur_map_uid = map_uids[i];
ret = -1;
continue;
}
/* already in primary storage */
continue;
}
}
return ret;
}
{
struct mdbox_purge_context *ctx;
struct seq_range_iter iter;
unsigned int i = 0;
bool deleted;
int ret;
if (mdbox_purge_get_primary_files(ctx) < 0)
ret = -1;
else {
/* add files that can be altmoved */
if (mdbox_altmove_add_files(ctx) < 0)
ret = -1;
}
}
while (ret == 0 &&
ret = -1;
} else {
ret = -1;
}
} T_END;
/* purging found corrupted files */
(void)mdbox_storage_rebuild(storage);
ret = -1;
}
return ret;
}