index/dbox-multi/mdbox-purge.c

	mdbox-purge.c revision 50854b51d6408653be4dd90bd0427978df25c2b7
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch/* Copyright (c) 2007-2011 Dovecot authors, see the included COPYING file */
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "lib.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "array.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "istream.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "ostream.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "str.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "hash.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "dbox-attachment.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mdbox-storage.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mdbox-storage-rebuild.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mdbox-file.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mdbox-map.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include "mdbox-sync.h"
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include <stdlib.h>
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen#include <dirent.h>
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen/*
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen   Altmoving works like:
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen   1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen      code and map UID's alt-refcount is updated. It won't be written to disk.
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen   2. mdbox_purge() is called, which checks if map UID's refcount equals
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen      to its alt-refcount. If it does, it's moved to alt storage. Moving to
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen      primary storage is done if _ALT flag was removed from any message.
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen*/
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainenenum mdbox_msg_action {
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen    MDBOX_MSG_ACTION_MOVE_TO_ALT = 1,
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen    MDBOX_MSG_ACTION_MOVE_FROM_ALT
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen};
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainenstruct mdbox_purge_context {
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    pool_t pool;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    struct mdbox_storage *storage;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    uint32_t lowest_primary_file_id;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    /* list of file_ids that exist in primary storage. this list is looked
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen       up while there is no locking, so it may not be accurate anymore by
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen       the time it's used. */
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    ARRAY_TYPE(seq_range) primary_file_ids;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    /* list of file_ids that we need to purge */
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    ARRAY_TYPE(seq_range) purge_file_ids;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    /* map_uid => mdbox_msg_action */
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    struct hash_table *altmoves;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    bool have_altmoves;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    struct mdbox_map_atomic_context *atomic;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    struct mdbox_map_append_context *append_ctx;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen};
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic int mdbox_map_file_msg_offset_cmp(const void *p1, const void *p2)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen{
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    const struct mdbox_map_file_msg *m1 = p1, *m2 = p2;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (m1->offset < m2->offset)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return -1;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    else if (m1->offset > m2->offset)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return 1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    else
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return 0;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen}
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainenstatic int
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmdbox_file_read_metadata_hdr(struct dbox_file *file,
5430411539347357344f6bf2552731b62f0a14fdTimo Sirainen                 struct dbox_metadata_header *meta_hdr_r)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen{
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    const unsigned char *data;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    size_t size;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    int ret;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    ret = i_stream_read_data(file->input, &data, &size,
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen                 sizeof(*meta_hdr_r));
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (ret <= 0) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        i_assert(ret == -1);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        if (file->input->stream_errno == 0) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            dbox_file_set_corrupted(file, "missing metadata");
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            return 0;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        mail_storage_set_critical(&file->storage->storage,
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            "read(%s) failed: %m", file->cur_path);
5430411539347357344f6bf2552731b62f0a14fdTimo Sirainen        return -1;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    memcpy(meta_hdr_r, data, sizeof(*meta_hdr_r));
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    if (memcmp(meta_hdr_r->magic_post, DBOX_MAGIC_POST,
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen           sizeof(meta_hdr_r->magic_post)) != 0) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        dbox_file_set_corrupted(file, "invalid metadata magic");
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        return 0;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    i_stream_skip(file->input, sizeof(*meta_hdr_r));
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    return 1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen}
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic int
e106213aebb4170904697814c168548e1ba97de9Timo Sirainenmdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen{
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    struct dbox_metadata_header meta_hdr;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    const char *line;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    size_t buf_size;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    int ret;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return ret;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    o_stream_send(output, &meta_hdr, sizeof(meta_hdr));
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    buf_size = i_stream_get_max_buffer_size(file->input);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    i_stream_set_max_buffer_size(file->input, 0);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    while ((line = i_stream_read_next_line(file->input)) != NULL) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        if (*line == '\0') {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            /* end of metadata */
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen            break;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        o_stream_send_str(output, line);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        o_stream_send(output, "\n", 1);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    i_stream_set_max_buffer_size(file->input, buf_size);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (line == NULL) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        dbox_file_set_corrupted(file, "missing end-of-metadata line");
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return 0;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    o_stream_send(output, "\n", 1);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    return 1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen}
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
5430411539347357344f6bf2552731b62f0a14fdTimo Sirainenstatic int
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen               ARRAY_TYPE(mail_attachment_extref) *extrefs)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen{
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    struct dbox_metadata_header meta_hdr;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    const char *line;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    size_t buf_size;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    int ret;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    /* skip and ignore the header */
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen        return ret;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    buf_size = i_stream_get_max_buffer_size(file->input);
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    i_stream_set_max_buffer_size(file->input, 0);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    while ((line = i_stream_read_next_line(file->input)) != NULL) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        if (*line == '\0') {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            /* end of metadata */
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            break;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        }
5430411539347357344f6bf2552731b62f0a14fdTimo Sirainen        if (*line == DBOX_METADATA_EXT_REF) {
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen            if (!dbox_attachment_parse_extref(line+1, ext_refs_pool,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen                              extrefs)) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen                i_warning("%s: Ignoring corrupted extref: %s",
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen                      file->cur_path, line);
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen            }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    i_stream_set_max_buffer_size(file->input, buf_size);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (line == NULL) {
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        dbox_file_set_corrupted(file, "missing end-of-metadata line");
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return 0;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    return 1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen}
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenstatic bool
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmdbox_purge_want_altpath(struct mdbox_purge_context *ctx, uint32_t map_uid)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen{
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    enum mdbox_msg_action action;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    void *value;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    if (!ctx->have_altmoves)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return FALSE;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid));
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (value == NULL)
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return FALSE;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    action = POINTER_CAST_TO(value, enum mdbox_msg_action);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    return action == MDBOX_MSG_ACTION_MOVE_TO_ALT;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen}
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainenstatic int
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainenmdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen             const struct mdbox_map_file_msg *msg)
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen{
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    struct dbox_file_append_context *out_file_append;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    struct istream *input;
6b8f4863bb2b0938d40f774122baf6528a833ea0Timo Sirainen    struct ostream *output;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    enum mdbox_map_append_flags append_flags;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    uoff_t msg_size;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    off_t ret;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    int read_errno;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (ctx->append_ctx == NULL)
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        ctx->append_ctx = mdbox_map_append_begin(ctx->atomic);
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    append_flags = !mdbox_purge_want_altpath(ctx, msg->map_uid) ? 0 :
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen        DBOX_MAP_APPEND_FLAG_ALT;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    msg_size = file->msg_header_size + file->cur_physical_size;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    if (mdbox_map_append_next(ctx->append_ctx, file->cur_physical_size,
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen                  append_flags, &out_file_append, &output) < 0)
4c9294fcb17fb12326ff5db29e2f882cf4bd7fedTimo Sirainen        return -1;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    i_assert(file != out_file_append->file);
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    input = i_stream_create_limit(file->input, msg_size);
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    ret = o_stream_send_istream(output, input);
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    read_errno = input->stream_errno;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    i_stream_unref(&input);
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (read_errno != 0) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        errno = read_errno;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        mail_storage_set_critical(&file->storage->storage,
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen            "read(%s) failed: %m", file->cur_path);
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        return -1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    if (output->stream_errno != 0) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        errno = output->stream_errno;
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen        mail_storage_set_critical(&file->storage->storage,
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen                      "write(%s) failed: %m",
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen                      out_file_append->file->cur_path);
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        return -1;
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen    }
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen    if (ret != (off_t)msg_size) {
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        i_assert(ret < (off_t)msg_size);
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        i_assert(i_stream_is_eof(file->input));
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        dbox_file_set_corrupted(file, "truncated message at EOF");
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        return 0;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    }
a8ebb72c0fba1a6a71104e530bf5903d5f149351Timo Sirainen
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    /* copy metadata */
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    if ((ret = mdbox_file_metadata_copy(file, output)) <= 0)
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        return ret;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    mdbox_map_append_finish(ctx->append_ctx);
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    return 1;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen}
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainenstatic int
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainenmdbox_file_purge_check_refcounts(struct mdbox_purge_context *ctx,
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen                 const ARRAY_TYPE(mdbox_map_file_msg) *msgs_arr)
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen{
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    struct mdbox_map *map = ctx->storage->map;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    struct mdbox_map_mail_index_record rec;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    uint16_t refcount;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    const struct mdbox_map_file_msg *msgs;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen    unsigned int i, count;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    int ret;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    if (mdbox_map_atomic_lock(ctx->atomic) < 0)
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        return -1;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen
18a2214eedb08d043277cf1d3e75c45762014663Timo Sirainen    msgs = array_get(msgs_arr, &count);
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    for (i = 0; i < count; i++) {
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        if (msgs[i].refcount != 0)
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen            continue;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        ret = mdbox_map_lookup_full(map, msgs[i].map_uid, &rec,
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen                        &refcount);
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        if (ret <= 0) {
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen            if (ret < 0)
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen                return -1;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen            mdbox_map_set_corrupted(map,
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen                "Purging unexpectedly lost map_uid=%u",
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen                msgs[i].map_uid);
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen            return -1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen        }
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen        if (refcount > 0)
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen            return 0;
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    }
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    return 1;
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen}
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainenstatic int
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainenmdbox_purge_attachments(struct mdbox_purge_context *ctx,
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen            const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr)
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen{
f55b07916c82db8b915b28252e44ce6fb2bd3080Timo Sirainen    struct dbox_storage *storage = &ctx->storage->storage;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    const struct mail_attachment_extref *extref;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    int ret = 0;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    array_foreach(extrefs_arr, extref) {
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen        if (index_attachment_delete(&storage->storage,
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen                        storage->attachment_fs,
e106213aebb4170904697814c168548e1ba97de9Timo Sirainen                        extref->path) < 0)
99df8a838cd9c5257ea5a2554383a9a999191e38Pali Rohár            ret = -1;
99df8a838cd9c5257ea5a2554383a9a999191e38Pali Rohár    }
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen    return ret;
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen}
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainenstatic int
99df8a838cd9c5257ea5a2554383a9a999191e38Pali Rohármdbox_file_purge(struct mdbox_purge_context *ctx, struct dbox_file *file,
a1973d0f171b027f9a7c642bc1c2134293731e1cTimo Sirainen         uint32_t file_id)
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen{
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    struct stat st;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    ARRAY_TYPE(mdbox_map_file_msg) msgs_arr;
baf3e87e186453fda13bd21f7cbcb2efc8492e8bTimo Sirainen    const struct mdbox_map_file_msg *msgs;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    ARRAY_TYPE(seq_range) expunged_map_uids;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    ARRAY_TYPE(uint32_t) copied_map_uids;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    ARRAY_TYPE(mail_attachment_extref) ext_refs;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    pool_t ext_refs_pool;
39025a2dabfcfaeee3790988b9ea00d19887a3d3Timo Sirainen    unsigned int i, count;
    uoff_t offset;
    int ret;

    i_assert(ctx->atomic == NULL);
    i_assert(ctx->append_ctx == NULL);

    if ((ret = dbox_file_try_lock(file)) <= 0)
        return ret;

    /* make sure the file still exists. another process may have already
       deleted it. */
    if (stat(file->cur_path, &st) < 0) {
        dbox_file_unlock(file);
        if (errno == ENOENT)
            return 0;

        mail_storage_set_critical(&file->storage->storage,
            "stat(%s) failed: %m", file->cur_path);
        return -1;
    }

    /* get list of map UIDs that exist in this file (again has to be done
       after locking) */
    i_array_init(&msgs_arr, 128);
    if (mdbox_map_get_file_msgs(dstorage->map, file_id,
                    &msgs_arr) < 0) {
        array_free(&msgs_arr);
        dbox_file_unlock(file);
        return -1;
    }
    /* sort messages by their offset */
    array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp);

    ext_refs_pool = pool_alloconly_create("mdbox purge ext refs", 1024);
    ctx->atomic = mdbox_map_atomic_begin(ctx->storage->map);
    msgs = array_get(&msgs_arr, &count);
    i_array_init(&ext_refs, 32);
    i_array_init(&copied_map_uids, I_MIN(count, 1));
    i_array_init(&expunged_map_uids, I_MIN(count, 1));
    offset = file->file_header_size;
    for (i = 0; i < count; i++) {
        if ((ret = dbox_file_seek(file, offset)) <= 0)
            break;

        if (msgs[i].offset != offset) {
            /* map doesn't match file's actual contents */
            dbox_file_set_corrupted(file,
                "purging found mismatched offsets "
                "(%"PRIuUOFF_T" vs %u, %u/%u)",
                offset, msgs[i].offset, i, count);
            ret = 0;
            break;
        }

        if (msgs[i].refcount == 0) {
            /* skip over expunged message */
            i_stream_seek(file->input, offset +
                      file->msg_header_size +
                      file->cur_physical_size);
            /* skip metadata */
            ret = mdbox_metadata_get_extrefs(file, ext_refs_pool,
                             &ext_refs);
            if (ret <= 0)
                break;
            seq_range_array_add(&expunged_map_uids, 0,
                        msgs[i].map_uid);
        } else {
            /* non-expunged message. write it to output file. */
            i_stream_seek(file->input, offset);
            ret = mdbox_purge_save_msg(ctx, file, &msgs[i]);
            if (ret <= 0)
                break;
            array_append(&copied_map_uids, &msgs[i].map_uid, 1);
        }
        offset = file->input->v_offset;
    }
    if (offset != (uoff_t)st.st_size && ret > 0) {
        /* file has more messages than what map tells us */
        dbox_file_set_corrupted(file,
            "more messages available than in map "
            "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size);
        ret = 0;
    }

    if (ret <= 0)
        ret = -1;
    else {
        /* it's possible that one of the messages we purged was
           just copied to another mailbox. the only way to prevent that
           would be to keep map locked during the purge, but that could
           keep it locked for too long. instead we'll check here if
           there are such copies, and if there are cancel this file's
           purge. */
        ret = mdbox_file_purge_check_refcounts(ctx, &msgs_arr);
    }
    array_free(&msgs_arr); msgs = NULL;

    if (ret <= 0) {
        /* failed */
    } else if (ctx->append_ctx == NULL) {
        /* everything purged from this file */
        ret = 1;
    } else {
        /* assign new file_id + offset to moved messages */
        if (mdbox_map_append_move(ctx->append_ctx, &copied_map_uids,
                      &expunged_map_uids) < 0 ||
            mdbox_map_append_commit(ctx->append_ctx) < 0)
            ret = -1;
        else
            ret = 1;
    }
    if (ctx->append_ctx != NULL)
        mdbox_map_append_free(&ctx->append_ctx);
    (void)mdbox_map_atomic_finish(&ctx->atomic);

    /* unlink only after unlocking map, so readers don't see it
       temporarily vanished */
    if (ret > 0) {
        (void)dbox_file_unlink(file);
        mdbox_map_remove_file_id(ctx->storage->map, file_id);
    } else {
        dbox_file_unlock(file);
    }
    array_free(&copied_map_uids);
    array_free(&expunged_map_uids);

    (void)mdbox_purge_attachments(ctx, &ext_refs);
    array_free(&ext_refs);
    pool_unref(&ext_refs_pool);
    return ret;
}

void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt)
{
    struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)mail->box;
    ARRAY_TYPE(uint32_t) *dest;
    uint32_t map_uid;

    /* we'll assume here that alt flag won't be changed multiple times
       for the same mail. it shouldn't happen with current code, and
       checking for it would just slow down the code.

       so the way it works currently is just that map_uids are added to
       an array, which is later sorted and processed further. note that
       it's still possible that the same map_uid exists in the array
       multiple times. */
    if (mdbox_mail_lookup(mbox, mbox->box.view, mail->seq, &map_uid) < 0)
        return;

    dest = move_to_alt ? &mbox->storage->move_to_alt_map_uids :
        &mbox->storage->move_from_alt_map_uids;

    if (!array_is_created(dest))
        i_array_init(dest, 256);
    array_append(dest, &map_uid, 1);
}

static struct mdbox_purge_context *
mdbox_purge_alloc(struct mdbox_storage *storage)
{
    struct mdbox_purge_context *ctx;
    pool_t pool;

    pool = pool_alloconly_create("mdbox purge context", 1024*32);
    ctx = p_new(pool, struct mdbox_purge_context, 1);
    ctx->pool = pool;
    ctx->storage = storage;
    ctx->lowest_primary_file_id = (uint32_t)-1;
    i_array_init(&ctx->primary_file_ids, 64);
    i_array_init(&ctx->purge_file_ids, 64);
    ctx->altmoves = hash_table_create(default_pool, pool, 0, NULL, NULL);
    return ctx;
}

static void mdbox_purge_free(struct mdbox_purge_context **_ctx)
{
    struct mdbox_purge_context *ctx = *_ctx;

    *_ctx = NULL;

    hash_table_destroy(&ctx->altmoves);
    array_free(&ctx->primary_file_ids);
    array_free(&ctx->purge_file_ids);
    pool_unref(&ctx->pool);
}

static int mdbox_purge_get_primary_files(struct mdbox_purge_context *ctx)
{
    struct mdbox_storage *dstorage = ctx->storage;
    struct mail_storage *storage = &dstorage->storage.storage;
    DIR *dir;
    struct dirent *d;
    string_t *path;
    unsigned int file_id, dir_len;
    int ret = 0;

    if (!array_is_created(&dstorage->move_to_alt_map_uids) &&
        !array_is_created(&dstorage->move_from_alt_map_uids)) {
        /* we don't need to do alt moving, don't bother getting list
           of primary files */
        return 0;
    }

    dir = opendir(dstorage->storage_dir);
    if (dir == NULL) {
        if (errno == ENOENT) {
            /* no storage directory at all yet */
            return 0;
        }
        mail_storage_set_critical(storage,
            "opendir(%s) failed: %m", dstorage->storage_dir);
        return -1;
    }

    path = t_str_new(256);
    str_append(path, dstorage->storage_dir);
    str_append_c(path, '/');
    dir_len = str_len(path);

    for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
        if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
                strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
            continue;
        if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
                  &file_id) < 0)
            continue;

        str_truncate(path, dir_len);
        str_append(path, d->d_name);
        seq_range_array_add(&ctx->primary_file_ids, 0, file_id);
    }
    if (array_count(&ctx->primary_file_ids) > 0) {
        const struct seq_range *range =
            array_idx(&ctx->primary_file_ids, 0);
        ctx->lowest_primary_file_id = range[0].seq1;
    }
    if (errno != 0) {
        mail_storage_set_critical(storage,
            "readdir(%s) failed: %m", dstorage->storage_dir);
        ret = -1;
    }
    if (closedir(dir) < 0) {
        mail_storage_set_critical(storage,
            "closedir(%s) failed: %m", dstorage->storage_dir);
        ret = -1;
    }
    return ret;
}

static int uint32_t_cmp(const uint32_t *u1, const uint32_t *u2)
{
    if (*u1 < *u2)
        return -1;
    if (*u1 > *u2)
        return 1;
    return 0;
}

static int mdbox_altmove_add_files(struct mdbox_purge_context *ctx)
{
    struct mdbox_storage *dstorage = ctx->storage;
    const uint32_t *map_uids;
    unsigned int i, count, alt_refcount = 0;
    struct mdbox_map_mail_index_record cur_rec;
    uint32_t cur_map_uid;
    uint16_t cur_refcount = 0;
    uoff_t offset;
    int ret = 0;

    /* first add move-to-alt actions */
    if (array_is_created(&dstorage->move_to_alt_map_uids)) {
        array_sort(&dstorage->move_to_alt_map_uids, uint32_t_cmp);
        map_uids = array_get(&dstorage->move_to_alt_map_uids, &count);
    } else {
        map_uids = NULL;
        count = 0;
    }
    cur_map_uid = 0;
    for (i = 0; i < count; i++) {
        if (cur_map_uid != map_uids[i]) {
            cur_map_uid = map_uids[i];
            if (mdbox_map_lookup_full(dstorage->map, cur_map_uid,
                          &cur_rec, &cur_refcount) < 0) {
                cur_refcount = (uint16_t)-1;
                ret = -1;
            }
            alt_refcount = 1;
        } else {
            alt_refcount++;
        }

        if (alt_refcount == cur_refcount &&
            seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) {
            /* all instances marked as moved to alt storage */
            hash_table_insert(ctx->altmoves,
                POINTER_CAST(cur_map_uid),
                POINTER_CAST(MDBOX_MSG_ACTION_MOVE_TO_ALT));
            seq_range_array_add(&ctx->purge_file_ids, 0,
                        cur_rec.file_id);
        }
    }

    /* next add move-from-alt actions. they override move-to-alt actions
       in case there happen to be any conflicts (shouldn't). only a single
       move-from-alt record is needed to do the move. */
    if (array_is_created(&dstorage->move_from_alt_map_uids))
        map_uids = array_get(&dstorage->move_from_alt_map_uids, &count);
    else {
        map_uids = NULL;
        count = 0;
    }
    cur_map_uid = 0;
    for (i = 0; i < count; i++) {
        if (cur_map_uid == map_uids[i])
            continue;
        cur_map_uid = map_uids[i];

        if (mdbox_map_lookup(dstorage->map, cur_map_uid,
                     &cur_rec.file_id, &offset) < 0) {
            ret = -1;
            continue;
        }
        if (seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) {
            /* already in primary storage */
            continue;
        }

        hash_table_insert(ctx->altmoves, POINTER_CAST(cur_map_uid),
                  POINTER_CAST(MDBOX_MSG_ACTION_MOVE_FROM_ALT));
        seq_range_array_add(&ctx->purge_file_ids, 0, cur_rec.file_id);
    }
    ctx->have_altmoves = hash_table_count(ctx->altmoves) > 0;
    return ret;
}

int mdbox_purge(struct mail_storage *_storage)
{
    struct mdbox_storage *storage = (struct mdbox_storage *)_storage;
    struct mdbox_purge_context *ctx;
    struct dbox_file *file;
    struct seq_range_iter iter;
    unsigned int i = 0;
    uint32_t file_id;
    bool deleted;
    int ret;

    ctx = mdbox_purge_alloc(storage);
    ret = mdbox_map_get_zero_ref_files(storage->map, &ctx->purge_file_ids);
    if (storage->alt_storage_dir != NULL) {
        if (mdbox_purge_get_primary_files(ctx) < 0)
            ret = -1;
        else {
            /* add files that can be altmoved */
            if (mdbox_altmove_add_files(ctx) < 0)
                ret = -1;
        }
    }

    seq_range_array_iter_init(&iter, &ctx->purge_file_ids); i = 0;
    while (ret == 0 &&
           seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN {
        file = mdbox_file_init(storage, file_id);
        if (dbox_file_open(file, &deleted) > 0 && !deleted) {
            if (mdbox_file_purge(ctx, file, file_id) < 0)
                ret = -1;
        } else {
            mdbox_map_remove_file_id(storage->map, file_id);
        }
        dbox_file_unref(&file);
    } T_END;
    mdbox_purge_free(&ctx);

    if (storage->corrupted) {
        /* purging found corrupted files */
        (void)mdbox_storage_rebuild(storage);
        ret = -1;
    }
    return ret;
}