mdbox-map.c revision b5d17f9b1224f496f18b45579a4253df3ae0dbba
2e37d45867d081db150ab78dad303b9077aea24fTimo Sirainen/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "lib.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "array.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "hash.h"
472369cba85d9f7c995dda60e7cd01d78b4a960aTimo Sirainen#include "ostream.h"
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen#include "mkdir-parents.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "mdbox-storage.h"
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen#include "mdbox-file.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "mdbox-map-private.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include <stdlib.h>
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include <dirent.h>
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen#define MAX_BACKWARDS_LOOKUPS 10
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10)
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen#define DBOX_FORCE_PURGE_MIN_RATIO 0.5
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen#define MAP_STORAGE(map) (&(map)->storage->storage.storage)
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstruct dbox_map_transaction_context {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct dbox_map *map;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_transaction *trans;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct mail_index_sync_ctx *sync_ctx;
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen unsigned int changed:1;
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen unsigned int success:1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen};
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenvoid dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen va_list args;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen map->storage->storage.files_corrupted = TRUE;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen va_start(args, format);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen mail_storage_set_critical(MAP_STORAGE(map),
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen "dbox map %s corrupted: %s",
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen map->index->filepath,
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen t_strdup_vprintf(format, args));
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen va_end(args);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen}
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainenstruct dbox_map *
de486b59018016977015ef42e6071155b60e82e1Timo Sirainendbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen const char *path)
220e21750948941dc6e33b8f11b552fa21d7f81eTimo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct dbox_map *map;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen gid_t tmp_gid;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen const char *tmp_origin;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen map = i_new(struct dbox_map, 1);
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen map->storage = storage;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen map->set = storage->set;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen map->path = i_strdup(path);
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen map->index = mail_index_alloc(path, MDBOX_GLOBAL_INDEX_PREFIX);
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen map->map_ext_id = mail_index_ext_register(map->index, "map",
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen sizeof(struct dbox_map_mail_index_header),
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen sizeof(struct dbox_map_mail_index_record),
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen sizeof(uint32_t));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen sizeof(uint16_t), sizeof(uint16_t));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen map->created_uid_validity = ioloop_time;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mailbox_list_get_permissions(root_list, NULL, &map->create_mode,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen &map->create_gid, &map->create_gid_origin);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mailbox_list_get_dir_permissions(root_list, NULL, &map->create_dir_mode,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen &tmp_gid, &tmp_origin);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mail_index_set_permissions(map->index, map->create_mode,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen map->create_gid, map->create_gid_origin);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return map;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenvoid dbox_map_deinit(struct dbox_map **_map)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct dbox_map *map = *_map;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen *_map = NULL;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen if (map->view != NULL) {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen mail_index_view_close(&map->view);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mail_index_close(map->index);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mail_index_free(&map->index);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen i_free(map->path);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen i_free(map);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic int dbox_map_mkdir_storage(struct dbox_map *map)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if (mkdir_parents_chgrp(map->path, map->create_dir_mode,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen map->create_gid, map->create_gid_origin) < 0 &&
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen errno != EEXIST) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mail_storage_set_critical(MAP_STORAGE(map),
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen "mkdir(%s) failed: %m", map->path);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen return 0;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen}
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainenint dbox_map_open(struct dbox_map *map, bool create_missing)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen{
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen enum mail_index_open_flags open_flags;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen int ret;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen if (map->view != NULL) {
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen /* already opened */
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen return 0;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen }
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY |
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_storage_settings_to_index_flags(MAP_STORAGE(map)->set);
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen if (create_missing) {
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if (dbox_map_mkdir_storage(map) < 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ret = mail_index_open(map->index, open_flags,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen MAP_STORAGE(map)->set->parsed_lock_method);
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen if (ret < 0) {
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_reset_error(map->index);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (ret == 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen /* index not found - for now just return failure */
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen map->view = mail_index_view_open(map->index);
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen return 0;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen}
6600c05e2ab38e9f662582b63c56b0c980a03748Timo Sirainen
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainenint dbox_map_refresh(struct dbox_map *map)
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen{
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen struct mail_index_view_sync_ctx *ctx;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen bool delayed_expunges;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen /* some open files may have read partially written mails. now that
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen map syncing makes the new mails visible, we need to make sure the
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen partial data is flushed out of memory */
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen mdbox_files_sync_input(map->storage);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen if (mail_index_refresh(map->view->index) < 0) {
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_reset_error(map->index);
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ctx = mail_index_view_sync_begin(map->view,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_reset_error(map->index);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
3fe67ec75ccae1230bb9eb9f16affc48377f6441Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainenstatic int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq,
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen uint32_t *file_id_r, uoff_t *offset_r,
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen uoff_t *size_r)
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen{
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen const struct dbox_map_mail_index_record *rec;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen const void *data;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen uint32_t uid;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen bool expunged;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_lookup_ext(map->view, seq, map->map_ext_id,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen &data, &expunged);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen rec = data;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (rec == NULL || rec->file_id == 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_lookup_uid(map->view, seq, &uid);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen dbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid);
27586e4785d56aeb76e1fd96af8db799688dc64aTimo Sirainen return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen *file_id_r = rec->file_id;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen *offset_r = rec->offset;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen *size_r = rec->size;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenstatic int
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainendbox_map_get_seq(struct dbox_map *map, uint32_t map_uid, uint32_t *seq_r)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) {
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk /* not found - try again after a refresh */
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk if (dbox_map_refresh(map) < 0)
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk return -1;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk if (!mail_index_lookup_seq(map->view, map_uid, seq_r))
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk return 0;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk }
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk return 1;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk}
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainenint dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen uint32_t *file_id_r, uoff_t *offset_r)
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen{
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen uint32_t seq;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uoff_t size;
5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen int ret;
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen if (dbox_map_open(map, TRUE) < 0)
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen return -1;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return ret;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r, &size) < 0)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return -1;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return 1;
9abf5be0962538e1f6f5c73c838ff677341da0c9Timo Sirainen}
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainenint dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view,
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen uint32_t seq, struct dbox_mail_lookup_rec *rec_r)
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen{
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen const uint16_t *ref16_p;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen const void *data;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen bool expunged;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen memset(rec_r, 0, sizeof(*rec_r));
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen mail_index_lookup_uid(view, seq, &rec_r->map_uid);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen mail_index_lookup_ext(view, seq, map->map_ext_id, &data, &expunged);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen if (data == NULL) {
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen dbox_map_set_corrupted(map, "missing map extension");
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen return -1;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen }
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen memcpy(&rec_r->rec, data, sizeof(rec_r->rec));
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen mail_index_lookup_ext(view, seq, map->ref_ext_id, &data, &expunged);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (data == NULL) {
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen dbox_map_set_corrupted(map, "missing ref extension");
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen return -1;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen }
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen ref16_p = data;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen rec_r->refcount = *ref16_p;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen return 0;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen}
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen
5c597df6aa8d81de4053c6986fab7739f3b44b20Timo Sirainenint dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id,
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen ARRAY_TYPE(dbox_map_file_msg) *recs)
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen{
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen const struct mail_index_header *hdr;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen struct dbox_mail_lookup_rec rec;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen struct dbox_map_file_msg msg;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen uint32_t seq;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (dbox_map_refresh(map) < 0)
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen return -1;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen hdr = mail_index_get_header(map->view);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen memset(&msg, 0, sizeof(msg));
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen if (dbox_map_view_lookup_rec(map, map->view, seq, &rec) < 0)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return -1;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (rec.rec.file_id == file_id) {
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen msg.map_uid = rec.map_uid;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen msg.offset = rec.rec.offset;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen msg.refcount = rec.refcount;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen array_append(recs, &msg, 1);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen }
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen }
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return 0;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen}
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainenint dbox_map_get_zero_ref_files(struct dbox_map *map,
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen ARRAY_TYPE(seq_range) *file_ids_r)
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen{
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen const struct mail_index_header *hdr;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen const struct dbox_map_mail_index_record *rec;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen const uint16_t *ref16_p;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen const void *data;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen uint32_t seq;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen bool expunged;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (dbox_map_open(map, FALSE) < 0) {
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen /* some internal error */
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen return -1;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen }
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (dbox_map_refresh(map) < 0)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return -1;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen hdr = mail_index_get_header(map->view);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen &data, &expunged);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (data != NULL && !expunged) {
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen ref16_p = data;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (*ref16_p != 0)
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen continue;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen }
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_lookup_ext(map->view, seq, map->map_ext_id,
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen &data, &expunged);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (data != NULL && !expunged) {
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen rec = data;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen seq_range_array_add(file_ids_r, 0, rec->file_id);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen }
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen }
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen return 0;
5494a6bc149da8f02fd25c0434a9d612ac33f659Timo Sirainen}
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainenstruct dbox_map_transaction_context *
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainendbox_map_transaction_begin(struct dbox_map *map, bool external)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen{
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen struct dbox_map_transaction_context *ctx;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen enum mail_index_transaction_flags flags =
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen MAIL_INDEX_TRANSACTION_FLAG_FSYNC;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (external)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen ctx = i_new(struct dbox_map_transaction_context, 1);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen ctx->map = map;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (dbox_map_open(map, FALSE) == 0 &&
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen dbox_map_refresh(map) == 0)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen ctx->trans = mail_index_transaction_begin(map->view, flags);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return ctx;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainenstatic void
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainendbox_map_sync_handle(struct dbox_map *map, struct mail_index_sync_ctx *sync_ctx)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen{
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct mail_index_sync_rec sync_rec;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen uint32_t seq1, seq2;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen uoff_t offset1, offset2;
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2);
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen if (offset1 != offset2 || seq1 != seq2) {
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen /* something had crashed. need a full resync. */
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen i_warning("dbox %s: Inconsistency in map index "
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")",
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen map->path, seq1, offset1, seq2, offset2);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen map->storage->storage.files_corrupted = TRUE;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen } else {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen while (mail_index_sync_next(sync_ctx, &sync_rec)) ;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainenint dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx)
4a514fb20e04df397842cde11cc9ea92abfe9728Timo Sirainen{
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen struct dbox_map *map = ctx->map;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct mail_index_view *view;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct mail_index_transaction *sync_trans;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen int ret;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (!ctx->changed)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* use syncing to lock the transaction log, so that we always see
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen log's head_offset = tail_offset */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen ret = mail_index_sync_begin(map->index, &ctx->sync_ctx,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen &view, &sync_trans, 0);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (ret <= 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen i_assert(ret != 0);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_reset_error(map->index);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_transaction_rollback(&ctx->trans);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen return -1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen dbox_map_sync_handle(map, ctx->sync_ctx);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (mail_index_transaction_commit(&ctx->trans) < 0) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_reset_error(map->index);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen return -1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen ctx->success = TRUE;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen return 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen}
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenvoid dbox_map_transaction_set_failed(struct dbox_map_transaction_context *ctx)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen ctx->success = FALSE;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen}
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenvoid dbox_map_transaction_free(struct dbox_map_transaction_context **_ctx)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct dbox_map_transaction_context *ctx = *_ctx;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct dbox_map *map = ctx->map;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen *_ctx = NULL;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (ctx->success) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_reset_error(map->index);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen } else if (ctx->sync_ctx != NULL) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_sync_rollback(&ctx->sync_ctx);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (ctx->trans != NULL)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen mail_index_transaction_rollback(&ctx->trans);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_free(ctx);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen}
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainenint dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const ARRAY_TYPE(uint32_t) *map_uids, int diff)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen{
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen struct dbox_map *map = ctx->map;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen const uint32_t *uidp;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen unsigned int i, count;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen const void *data;
494a5de15db3b2806ab31d5ecc3e1c306ae14d06Timo Sirainen uint32_t seq;
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen bool expunged;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen int cur_diff;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen if (ctx->trans == NULL)
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen count = array_count(map_uids);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen for (i = 0; i < count; i++) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen uidp = array_idx(map_uids, i);
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen if (!mail_index_lookup_seq(map->view, *uidp, &seq)) {
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen /* we can't refresh map here since view has a
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen transaction open. */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen dbox_map_set_corrupted(map,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen "refcount update lost map_uid=%u", *uidp);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen return -1;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen }
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen &data, &expunged);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen cur_diff = data == NULL ? 0 : *((const uint16_t *)data);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen ctx->changed = TRUE;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen map->ref_ext_id, diff);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (cur_diff < 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen dbox_map_set_corrupted(map,
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen "map_uid=%u refcount too low", *uidp);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (cur_diff >= 32768) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* we're getting close to the 64k limit. fail early
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen to make it less likely that two processes increase
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen the refcount enough times to cross the limit */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen mail_storage_set_error(MAP_STORAGE(map),
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen MAIL_ERROR_NOTPOSSIBLE,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen "Message has been copied too many times");
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return 0;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen}
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainenint dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct dbox_map_transaction_context *map_trans;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const struct mail_index_header *hdr;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const struct dbox_map_mail_index_record *rec;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen const void *data;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen bool expunged;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen uint32_t seq;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen int ret = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* make sure the map is refreshed, otherwise we might be expunging
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen messages that have already been moved to other files. */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* we need a per-file transaction, otherwise we can't refresh the map */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen map_trans = dbox_map_transaction_begin(map, TRUE);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen hdr = mail_index_get_header(map->view);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_lookup_ext(map->view, seq, map->map_ext_id,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen &data, &expunged);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (data == NULL) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen dbox_map_set_corrupted(map, "missing map extension");
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen ret = -1;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen break;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen rec = data;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (rec->file_id == file_id) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen map_trans->changed = TRUE;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_expunge(map_trans->trans, seq);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (ret == 0)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen (void)dbox_map_transaction_commit(map_trans);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen dbox_map_transaction_free(&map_trans);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen return ret;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen}
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainenstruct dbox_map_append_context *
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainendbox_map_append_begin(struct dbox_map *map, enum dbox_map_append_flags flags)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen{
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen struct dbox_map_append_context *ctx;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ctx = i_new(struct dbox_map_append_context, 1);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ctx->map = map;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen ctx->flags = flags;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ctx->first_new_file_id = (uint32_t)-1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_array_init(&ctx->file_appends, 64);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_array_init(&ctx->files, 64);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_array_init(&ctx->appends, 128);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (dbox_map_open(ctx->map, TRUE) < 0)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ctx->failed = TRUE;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen else {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* refresh the map so we can try appending to the
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen latest files */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen (void)dbox_map_refresh(ctx->map);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen return ctx;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen}
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainenstatic time_t day_begin_stamp(unsigned int interval)
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct tm tm;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen time_t stamp;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen unsigned int unit = 1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (interval == 0)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* get the beginning of day/hour/minute depending on how large
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen the interval is */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen tm = *localtime(&ioloop_time);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (interval >= 60) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen tm.tm_sec = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen unit = 60;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (interval >= 3600) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen tm.tm_min = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen unit = 3600;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (interval >= 3600*24) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen tm.tm_hour = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen unit = 3600*24;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen stamp = mktime(&tm);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (stamp == (time_t)-1)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_panic("mktime(today) failed");
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return stamp - (interval - unit);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen}
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainenstatic bool dbox_try_open(struct dbox_map_append_context *ctx,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen struct dbox_file *file)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen bool want_altpath, notfound;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen want_altpath = (ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) != 0;
cb933f0a570a9cef5c975eadb818aa6b1002a269Timo Sirainen if (want_altpath) {
cb933f0a570a9cef5c975eadb818aa6b1002a269Timo Sirainen if (dbox_file_open(file, &notfound) <= 0)
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen return FALSE;
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen } else {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if (dbox_file_open_primary(file, &notfound) <= 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return FALSE;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen if (notfound)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return FALSE;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if (file->lock != NULL) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen /* already locked, we're possibly in the middle of purging it
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen in which case we really don't want to write there. */
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen return FALSE;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen }
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen if (dbox_file_is_in_alt(file) != want_altpath) {
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen /* different alt location than what we want, can't use it */
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen return FALSE;
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen }
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen return TRUE;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen}
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainenstatic bool
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainendbox_map_file_try_append(struct dbox_map_append_context *ctx,
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen uint32_t file_id, time_t stamp, uoff_t mail_size,
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct dbox_file_append_context **file_append_r,
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct ostream **output_r, bool *retry_later_r)
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen{
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct dbox_map *map = ctx->map;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct mdbox_storage *storage = map->storage;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct dbox_file *file;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct dbox_file_append_context *file_append;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen struct stat st;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen bool file_too_old = FALSE;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen int ret;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen *file_append_r = NULL;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen *output_r = NULL;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen *retry_later_r = FALSE;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen file = mdbox_file_init(storage, file_id);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (!dbox_try_open(ctx, file)) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen dbox_file_unref(&file);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return TRUE;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (file->create_time < stamp)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen file_too_old = TRUE;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen else if ((ret = dbox_file_try_lock(file)) <= 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* locking failed */
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen *retry_later_r = ret == 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen } else if (stat(file->cur_path, &st) < 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (errno != ENOENT)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_error("stat(%s) failed: %m", file->cur_path);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* the file was unlinked between opening and locking it. */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen } else {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen file_append = dbox_file_append_init(file);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (dbox_file_get_append_stream(file_append, output_r) <= 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* couldn't append to this file */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen } else if ((*output_r)->offset + mail_size > map->set->mdbox_rotate_size) {
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen /* file was too large after all */
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen } else {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen /* success */
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen *file_append_r = file_append;
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen return TRUE;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen }
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen dbox_file_append_rollback(&file_append);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen }
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen /* failure */
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen dbox_file_unlock(file);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen dbox_file_unref(&file);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen return !file_too_old;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen}
b82d6d7f02734007c129fa25bc876049c8d9bddeTimo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainenstatic bool
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainendbox_map_is_appending(struct dbox_map_append_context *ctx, uint32_t file_id)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen{
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen struct dbox_file_append_context *const *file_appends;
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen unsigned int i, count;
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen /* there shouldn't be many files open, don't bother with anything
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk faster. */
c78e7a94528078728cc639b26a1c83e11b4d7e1bTimo Sirainen file_appends = array_get(&ctx->file_appends, &count);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen for (i = 0; i < count; i++) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen struct mdbox_file *mfile =
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen (struct mdbox_file *)file_appends[i]->file;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen if (mfile->file_id == file_id)
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen return TRUE;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen }
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen return FALSE;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen}
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainenstatic struct dbox_file_append_context *
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainendbox_map_find_existing_append(struct dbox_map_append_context *ctx,
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen uoff_t mail_size, struct ostream **output_r)
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen{
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen struct dbox_map *map = ctx->map;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen struct dbox_file_append_context *const *file_appends, *append;
0910ea0672c0295c442eb686cc41e98656831f37Timo Sirainen struct mdbox_file *mfile;
0910ea0672c0295c442eb686cc41e98656831f37Timo Sirainen unsigned int i, count;
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen uoff_t append_offset;
e91543761d0b7b97a1dc28e036e44d76405545c2Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen if (mail_size >= map->set->mdbox_rotate_size)
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen return NULL;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen /* first try to use files already used in this append */
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen file_appends = array_get(&ctx->file_appends, &count);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen for (i = count; i > ctx->files_nonappendable_count; i--) {
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen append = file_appends[i-1];
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen append_offset = append->output->offset;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen if (append_offset + mail_size <= map->set->mdbox_rotate_size &&
93688bfedcfb2b9c02750b8d4d409123a386de5cTimo Sirainen dbox_file_get_append_stream(append, output_r) > 0)
93688bfedcfb2b9c02750b8d4d409123a386de5cTimo Sirainen return append;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen /* can't append to this file anymore. if we created this file,
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen close it so we don't waste fds. if we didn't, we can't close
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen it without also losing our lock too early. */
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen mfile = (struct mdbox_file *)append->file;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen if (mfile->file_id == 0 && dbox_file_append_flush(append) == 0)
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen dbox_file_close(append->file);
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen }
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen ctx->files_nonappendable_count = count;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen return NULL;
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen}
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainenstatic int
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainendbox_map_find_first_alt(struct dbox_map_append_context *ctx,
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen uint32_t *min_file_id_r, uint32_t *seq_r)
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen{
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen struct mdbox_storage *dstorage = ctx->map->storage;
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen struct mail_storage *storage = &dstorage->storage.storage;
fe813f74aaccb12f38e1bd9cd338c6a37fa646e5Timo Sirainen DIR *dir;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk struct dirent *d;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen const struct mail_index_header *hdr;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen uint32_t seq, file_id, min_file_id = -1U;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk uoff_t offset, size;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk int ret = 0;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk
c78e7a94528078728cc639b26a1c83e11b4d7e1bTimo Sirainen /* we want to quickly find the latest alt file, but we also want to
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen avoid accessing the alt storage as much as possible. so we'll do
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen this by finding the lowest numbered file (n) from primary storage.
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen hopefully one of n-[1..m] is appendable in alt storage. */
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen dir = opendir(dstorage->storage_dir);
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen if (dir == NULL) {
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen mail_storage_set_critical(storage,
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen "opendir(%s) failed: %m", dstorage->storage_dir);
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen return -1;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen }
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
17cf149e5f47183bfcc1503649dfb92a14f9dcd9Timo Sirainen if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen continue;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen file_id = strtoul(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
fea7b8b3fc182e415b1875d79587c0aa1adb09d7Timo Sirainen NULL, 10);
fea7b8b3fc182e415b1875d79587c0aa1adb09d7Timo Sirainen if (min_file_id > file_id)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen min_file_id = file_id;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (errno != 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen mail_storage_set_critical(storage,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen "readdir(%s) failed: %m", dstorage->storage_dir);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen ret = -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (closedir(dir) < 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen mail_storage_set_critical(storage,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen "closedir(%s) failed: %m", dstorage->storage_dir);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen ret = -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (ret < 0)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* find the newest message in alt storage from map view */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen hdr = mail_index_get_header(ctx->map->view);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen for (seq = hdr->messages_count; seq > 0; seq--) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (dbox_map_lookup_seq(ctx->map, seq, &file_id,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen &offset, &size) < 0)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (file_id < min_file_id)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen break;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen *min_file_id_r = min_file_id;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen *seq_r = seq;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainenstatic int
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainendbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen uoff_t mail_size,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct dbox_file_append_context **file_append_r,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct ostream **output_r)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen{
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct dbox_map *map = ctx->map;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen ARRAY_TYPE(seq_range) checked_file_ids;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen const struct mail_index_header *hdr;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen unsigned int backwards_lookup_count;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen uint32_t seq, seq1, uid, file_id, min_file_id;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen uoff_t offset, size;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen time_t stamp;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen bool retry_later;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (mail_size >= map->set->mdbox_rotate_size)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* try to find an existing appendable file */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen stamp = day_begin_stamp(map->set->mdbox_rotate_interval);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen hdr = mail_index_get_header(map->view);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen backwards_lookup_count = 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen t_array_init(&checked_file_ids, 16);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if ((ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) == 0)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen seq = hdr->messages_count;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen else {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* we want to save to alt storage. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (dbox_map_find_first_alt(ctx, &min_file_id, &seq) < 0)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen seq_range_array_add_range(&checked_file_ids,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen min_file_id, (uint32_t)-1);
be110f0a3888a24282f06820ec0b5068d2b8f906Timo Sirainen }
be110f0a3888a24282f06820ec0b5068d2b8f906Timo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen for (; seq > 0; seq--) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return -1;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (seq_range_exists(&checked_file_ids, file_id))
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen continue;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen seq_range_array_add(&checked_file_ids, 0, file_id);
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* we've wasted enough time here */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen break;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* first lookup: this should be enough usually, but we can't
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen be sure until after locking. also if messages were recently
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen moved, this message might not be the last one in the file. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (offset + size + mail_size >= map->set->mdbox_rotate_size)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen continue;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (dbox_map_is_appending(ctx, file_id)) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* already checked this */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen continue;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen mail_index_lookup_uid(map->view, seq, &uid);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (!dbox_map_file_try_append(ctx, file_id, stamp, mail_size,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen file_append_r, output_r,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen &retry_later)) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* file is too old. the rest of the files are too. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen break;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* NOTE: we've now refreshed map view. there are no guarantees
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen about sequences anymore. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (*file_append_r != NULL)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return 1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* FIXME: use retry_later somehow */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (uid == 1 ||
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen !mail_index_lookup_seq_range(map->view, 1, uid-1,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen &seq1, &seq))
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen break;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen seq++;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen return 0;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen}
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainenint dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct dbox_file_append_context **file_append_ctx_r,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct ostream **output_r)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen{
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct dbox_file *file;
e5aa8d4c9bdda79c2db03391ec45b37bf774df14Timo Sirainen struct dbox_map_append *append;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen struct dbox_file_append_context *file_append;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen bool existing;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen int ret;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (ctx->failed)
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen return -1;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen file_append = dbox_map_find_existing_append(ctx, mail_size, output_r);
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen if (file_append != NULL) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ret = 1;
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen existing = TRUE;
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen } else {
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen ret = dbox_map_find_appendable_file(ctx, mail_size,
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen &file_append, output_r);
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen existing = FALSE;
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if (ret > 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen file = file_append->file;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen else if (ret < 0)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen else {
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen /* create a new file */
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen file = (ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) == 0 ?
9f1a154d6f436e67f22f737f66bb148b502d4d2aTimo Sirainen mdbox_file_init(ctx->map->storage, 0) :
1c885b304f060e3ac4fe04195a2f53457d0ac99eTimo Sirainen mdbox_file_init_new_alt(ctx->map->storage);
1c885b304f060e3ac4fe04195a2f53457d0ac99eTimo Sirainen file_append = dbox_file_append_init(file);
1c885b304f060e3ac4fe04195a2f53457d0ac99eTimo Sirainen
1c885b304f060e3ac4fe04195a2f53457d0ac99eTimo Sirainen ret = dbox_file_get_append_stream(file_append, output_r);
1c885b304f060e3ac4fe04195a2f53457d0ac99eTimo Sirainen if (ret <= 0) {
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen i_assert(ret < 0);
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen dbox_file_append_rollback(&file_append);
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen dbox_file_unref(&file);
6d931bbce16786df431e9ae8201a78a95084316dTimo Sirainen return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen append = array_append_space(&ctx->appends);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen append->file_append = file_append;
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen append->offset = (*output_r)->offset;
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen append->size = (uint32_t)-1;
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen if (!existing) {
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen i_assert(file_append->first_append_offset == 0);
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen file_append->first_append_offset = file_append->output->offset;
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen array_append(&ctx->file_appends, &file_append, 1);
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen array_append(&ctx->files, &file, 1);
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen }
6b067886457724ca13d81e869240500c5bd5d7e2Timo Sirainen *file_append_ctx_r = file_append;
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen return 0;
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen}
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainenvoid dbox_map_append_finish(struct dbox_map_append_context *ctx)
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen{
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen struct dbox_map_append *appends;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen unsigned int count;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen uoff_t cur_offset;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen appends = array_get_modifiable(&ctx->appends, &count);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen i_assert(count > 0 && appends[count-1].size == (uint32_t)-1);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen cur_offset = appends[count-1].file_append->output->offset;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen i_assert(cur_offset >= appends[count-1].offset);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen appends[count-1].size = cur_offset - appends[count-1].offset;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen}
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainenstatic int
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainendbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view,
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen uint32_t *file_id_r)
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen{
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen const struct dbox_map_mail_index_header *hdr;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen const void *data;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen size_t data_size;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen if (data_size != sizeof(*hdr)) {
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen if (data_size != 0) {
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen dbox_map_set_corrupted(map, "hdr size=%"PRIuSIZE_T,
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen data_size);
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen return -1;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen }
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen /* first file */
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen *file_id_r = 1;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen } else {
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen hdr = data;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen *file_id_r = hdr->highest_file_id + 1;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen }
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen return 0;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen}
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainenstatic int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx,
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen bool separate_transaction)
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen{
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen struct dbox_file_append_context *const *file_appends;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen unsigned int i, count;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen uint32_t first_file_id, file_id;
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen int ret;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen /* start the syncing. we'll need it even if there are no file ids to
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen be assigned. */
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen ret = mail_index_sync_begin(ctx->map->index, &ctx->sync_ctx,
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen &ctx->sync_view, &ctx->sync_trans, 0);
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen if (ret <= 0) {
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen i_assert(ret != 0);
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen mail_storage_set_internal_error(MAP_STORAGE(ctx->map));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mail_index_reset_error(ctx->map->index);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return -1;
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen }
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen dbox_map_sync_handle(ctx->map, ctx->sync_ctx);
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) {
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen mail_index_sync_rollback(&ctx->sync_ctx);
5160580b0ec3f3288a320987abdf12a990f09df5Timo Sirainen return -1;
5160580b0ec3f3288a320987abdf12a990f09df5Timo Sirainen }
5160580b0ec3f3288a320987abdf12a990f09df5Timo Sirainen
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen /* assign file_ids for newly created files */
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen first_file_id = file_id;
5160580b0ec3f3288a320987abdf12a990f09df5Timo Sirainen file_appends = array_get(&ctx->file_appends, &count);
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen for (i = 0; i < count; i++) {
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen struct mdbox_file *mfile =
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen (struct mdbox_file *)file_appends[i]->file;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen if (dbox_file_append_flush(file_appends[i]) < 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ret = -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen break;
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen }
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen if (mfile->file_id == 0) {
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen if (mdbox_file_assign_file_id(mfile, file_id++) < 0) {
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen ret = -1;
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen break;
f97cacf16251b42f530c6a28686cc8c9aa7df3a2Timo Sirainen }
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen }
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen }
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen if (ret < 0) {
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen mail_index_sync_rollback(&ctx->sync_ctx);
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen return -1;
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen }
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen ctx->trans = !separate_transaction ? NULL :
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen mail_index_transaction_begin(ctx->map->view,
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen MAIL_INDEX_TRANSACTION_FLAG_FSYNC);
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen /* update the highest used file_id */
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen if (first_file_id != file_id) {
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen file_id--;
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans :
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen ctx->sync_trans,
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen ctx->map->map_ext_id,
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen 0, &file_id, sizeof(file_id));
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen }
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen return 0;
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen}
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainenint dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx,
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen uint32_t *first_map_uid_r,
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen uint32_t *last_map_uid_r)
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen{
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen const struct dbox_map_append *appends;
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen const struct mail_index_header *hdr;
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen struct dbox_map_mail_index_record rec;
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen unsigned int i, count;
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen ARRAY_TYPE(seq_range) uids;
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen const struct seq_range *range;
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen uint32_t seq;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen uint16_t ref16;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen int ret = 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (array_count(&ctx->appends) == 0) {
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen *first_map_uid_r = 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen *last_map_uid_r = 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen }
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (dbox_map_assign_file_ids(ctx, TRUE) < 0)
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return -1;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen /* append map records to index */
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen memset(&rec, 0, sizeof(rec));
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen ref16 = 1;
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen appends = array_get(&ctx->appends, &count);
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen for (i = 0; i < count; i++) {
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen struct mdbox_file *mfile =
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen (struct mdbox_file *)appends[i].file_append->file;
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen i_assert(appends[i].offset <= (uint32_t)-1);
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen i_assert(appends[i].size <= (uint32_t)-1);
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen rec.file_id = mfile->file_id;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen rec.offset = appends[i].offset;
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen rec.size = appends[i].size;
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen
6b8bcce7a001076605cb2cfdca8507204c88f376Timo Sirainen mail_index_append(ctx->trans, 0, &seq);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id,
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen &rec, NULL);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id,
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen &ref16, NULL);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen }
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen /* assign map UIDs for appended records */
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen hdr = mail_index_get_header(ctx->sync_view);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen t_array_init(&uids, 1);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_append_finish_uids(ctx->trans, hdr->next_uid, &uids);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen range = array_idx(&uids, 0);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen i_assert(range[0].seq2 - range[0].seq1 + 1 == count);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (hdr->uid_validity == 0) {
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen /* we don't really care about uidvalidity, but it can't be 0 */
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen uint32_t uid_validity = ioloop_time;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_update_header(ctx->trans,
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen offsetof(struct mail_index_header, uid_validity),
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen &uid_validity, sizeof(uid_validity), TRUE);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen }
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (mail_index_transaction_commit(&ctx->trans) < 0) {
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_storage_set_internal_error(MAP_STORAGE(ctx->map));
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_reset_error(ctx->map->index);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen *first_map_uid_r = range[0].seq1;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen *last_map_uid_r = range[0].seq2;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return ret;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen}
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainenint dbox_map_append_move(struct dbox_map_append_context *ctx,
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen const ARRAY_TYPE(uint32_t) *map_uids,
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen const ARRAY_TYPE(seq_range) *expunge_map_uids)
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen{
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen const struct dbox_map_append *appends;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen struct dbox_map_mail_index_record rec;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen struct seq_range_iter iter;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen const uint32_t *uids;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen unsigned int i, j, map_uids_count, appends_count;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen uint32_t uid, seq;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (dbox_map_assign_file_ids(ctx, FALSE) < 0)
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return -1;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen memset(&rec, 0, sizeof(rec));
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen appends = array_get(&ctx->appends, &appends_count);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen uids = array_get(map_uids, &map_uids_count);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen for (i = j = 0; i < map_uids_count; i++) {
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen struct mdbox_file *mfile =
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen (struct mdbox_file *)appends[j].file_append->file;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen i_assert(j < appends_count);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen rec.file_id = mfile->file_id;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen rec.offset = appends[j].offset;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen rec.size = appends[j].size;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen j++;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (!mail_index_lookup_seq(ctx->sync_view, uids[i], &seq))
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen i_unreached();
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_update_ext(ctx->sync_trans, seq,
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen ctx->map->map_ext_id, &rec, NULL);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen }
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen seq_range_array_iter_init(&iter, expunge_map_uids); i = 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen while (seq_range_array_iter_nth(&iter, i++, &uid)) {
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq))
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen i_unreached();
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_expunge(ctx->sync_trans, seq);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen }
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen}
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainenint dbox_map_append_commit(struct dbox_map_append_context *ctx)
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen{
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen struct dbox_map *map = ctx->map;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen struct dbox_file_append_context **file_appends;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen unsigned int i, count;
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen i_assert(ctx->trans == NULL);
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen file_appends = array_get_modifiable(&ctx->file_appends, &count);
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen for (i = 0; i < count; i++) {
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen if (dbox_file_append_commit(&file_appends[i]) < 0)
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen return -1;
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen }
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen if (ctx->sync_ctx != NULL) {
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_storage_set_internal_error(MAP_STORAGE(map));
c07e35cfc9587fc7589cbc1db0daeeb828456b2bTimo Sirainen mail_index_reset_error(map->index);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return -1;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen }
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen }
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen ctx->committed = TRUE;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return 0;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen}
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainenvoid dbox_map_append_free(struct dbox_map_append_context **_ctx)
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen{
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen struct dbox_map_append_context *ctx = *_ctx;
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen struct dbox_file_append_context **file_appends;
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen struct dbox_file **files;
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen unsigned int i, count;
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen *_ctx = NULL;
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen if (ctx->trans != NULL)
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen mail_index_transaction_rollback(&ctx->trans);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen if (ctx->sync_ctx != NULL)
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen mail_index_sync_rollback(&ctx->sync_ctx);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen file_appends = array_get_modifiable(&ctx->file_appends, &count);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen for (i = 0; i < count; i++) {
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen if (file_appends[i] != NULL)
407453d06e5622ac3bf099cc76bfcbfb5860b444Timo Sirainen dbox_file_append_rollback(&file_appends[i]);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen }
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen files = array_get_modifiable(&ctx->files, &count);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen for (i = 0; i < count; i++) {
547847191d4a95cd267fc846a1ca3c2111214719Timo Sirainen dbox_file_unlock(files[i]);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen dbox_file_unref(&files[i]);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen }
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen array_free(&ctx->appends);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen array_free(&ctx->file_appends);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen array_free(&ctx->files);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen i_free(ctx);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen}
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainenuint32_t dbox_map_get_uid_validity(struct dbox_map *map)
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen{
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen const struct mail_index_header *hdr;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen i_assert(map->view != NULL);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen hdr = mail_index_get_header(map->view);
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen if (hdr->uid_validity != 0)
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen return hdr->uid_validity;
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen /* refresh index in case it was just changed */
de486b59018016977015ef42e6071155b60e82e1Timo Sirainen (void)dbox_map_refresh(map);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen hdr = mail_index_get_header(map->view);
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen return hdr->uid_validity != 0 ? hdr->uid_validity :
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen map->created_uid_validity;
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen}
de4d9f77d6378a416c9963963eac5ac18b75ec0bTimo Sirainen