mdbox-purge.c revision 8776322310b57a11c52cfb1822f35cf18b095525
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2007-2014 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include "array.h"
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen#include "istream.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "ostream.h"
5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen#include "str.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "hash.h"
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include "dbox-attachment.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mdbox-storage.h"
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include "mdbox-storage-rebuild.h"
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include "mdbox-file.h"
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include "mdbox-map.h"
3ed2d0f6b5e67e2663d44489d9da3176823789a8Timo Sirainen#include "mdbox-sync.h"
65f8fb656051f1059f7b5a2da9c5555adcc30439Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include <stdlib.h>
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include <dirent.h>
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen/*
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen Altmoving works like:
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen 1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen code and map UID's alt-refcount is updated. It won't be written to disk.
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen 2. mdbox_purge() is called, which checks if map UID's refcount equals
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen to its alt-refcount. If it does, it's moved to alt storage. Moving to
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen primary storage is done if _ALT flag was removed from any message.
f7539a17ea306191b53b8f5e752e228937df9ec3Timo Sirainen*/
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2dd39e478269d6fb0bb26d12b394aa30ee965e38Timo Sirainenenum mdbox_msg_action {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen MDBOX_MSG_ACTION_MOVE_TO_ALT = 1,
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen MDBOX_MSG_ACTION_MOVE_FROM_ALT
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen};
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainenstruct mdbox_purge_context {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen pool_t pool;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen struct mdbox_storage *storage;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint32_t lowest_primary_file_id;
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen /* list of file_ids that exist in primary storage. this list is looked
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen up while there is no locking, so it may not be accurate anymore by
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen the time it's used. */
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen ARRAY_TYPE(seq_range) primary_file_ids;
3e564425db51f3921ce4de11859777135fdedd15Timo Sirainen /* list of file_ids that we need to purge */
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen ARRAY_TYPE(seq_range) purge_file_ids;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* uint32_t map_uid => enum mdbox_msg_action action */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen HASH_TABLE(void *, void *) altmoves;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen bool have_altmoves;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen struct mdbox_map_atomic_context *atomic;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen struct mdbox_map_append_context *append_ctx;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen};
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainenstatic int mdbox_map_file_msg_offset_cmp(const struct mdbox_map_file_msg *m1,
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen const struct mdbox_map_file_msg *m2)
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen{
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen if (m1->offset < m2->offset)
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen return -1;
dfaefeabae939803ceb8c503101e86b5496541d1Timo Sirainen else if (m1->offset > m2->offset)
dfaefeabae939803ceb8c503101e86b5496541d1Timo Sirainen return 1;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen else
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return 0;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainenstatic int
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenmdbox_file_read_metadata_hdr(struct dbox_file *file,
57a8c6a95e4bce3eeaba36985adb81c07dd683ffTimo Sirainen struct dbox_metadata_header *meta_hdr_r)
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen{
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen const unsigned char *data;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen size_t size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen ret = i_stream_read_data(file->input, &data, &size,
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen sizeof(*meta_hdr_r));
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen if (ret <= 0) {
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen i_assert(ret == -1);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen if (file->input->stream_errno == 0) {
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen dbox_file_set_corrupted(file, "missing metadata");
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen return 0;
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen }
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen mail_storage_set_critical(&file->storage->storage,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen "read(%s) failed: %m", file->cur_path);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen return -1;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen }
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
f1743785713e7632459d623d5df2108f4b93accbTimo Sirainen memcpy(meta_hdr_r, data, sizeof(*meta_hdr_r));
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (memcmp(meta_hdr_r->magic_post, DBOX_MAGIC_POST,
70ead6466f9baa8294e71fc2fba0a4f54f488b5eTimo Sirainen sizeof(meta_hdr_r->magic_post)) != 0) {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen dbox_file_set_corrupted(file, "invalid metadata magic");
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen return 0;
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen }
8d630c15a8ed6f85553467c3a231a273defca5f6Timo Sirainen i_stream_skip(file->input, sizeof(*meta_hdr_r));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 1;
ee116df08d0fdab703483e18fe8076b2ef9fd9d7Timo Sirainen}
c5ab90cfad9cc3e33bcb1baeb30ffc82a7b7053aTimo Sirainen
c5ab90cfad9cc3e33bcb1baeb30ffc82a7b7053aTimo Sirainenstatic int
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenmdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct dbox_metadata_header meta_hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *line;
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen size_t buf_size;
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen int ret;
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen return ret;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen o_stream_nsend(output, &meta_hdr, sizeof(meta_hdr));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buf_size = i_stream_get_max_buffer_size(file->input);
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen /* use unlimited line length for metadata */
cff1f182205e674285cf3ff446a0dcf7afea277dTimo Sirainen i_stream_set_max_buffer_size(file->input, (size_t)-1);
cff1f182205e674285cf3ff446a0dcf7afea277dTimo Sirainen while ((line = i_stream_read_next_line(file->input)) != NULL) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (*line == '\0') {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen /* end of metadata */
e03d986a74128f5ba30fcfda9f6e36578f5d8decTimo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen o_stream_nsend_str(output, line);
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen o_stream_nsend(output, "\n", 1);
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen }
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen i_stream_set_max_buffer_size(file->input, buf_size);
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen if (line == NULL) {
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen dbox_file_set_corrupted(file, "missing end-of-metadata line");
d3280fe317a4598c0868cc440e7a1191c06d0db3Timo Sirainen return 0;
d3280fe317a4598c0868cc440e7a1191c06d0db3Timo Sirainen }
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen o_stream_nsend(output, "\n", 1);
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen return 1;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainenstatic int
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainenmdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool,
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen ARRAY_TYPE(mail_attachment_extref) *extrefs)
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen{
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen struct dbox_metadata_header meta_hdr;
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen const char *line;
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen size_t buf_size;
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen int ret;
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen /* skip and ignore the header */
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen return ret;
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen
c979eeda1f46483d9c963e265786b701d7683d77Timo Sirainen buf_size = i_stream_get_max_buffer_size(file->input);
c979eeda1f46483d9c963e265786b701d7683d77Timo Sirainen /* use unlimited line length for metadata */
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen i_stream_set_max_buffer_size(file->input, (size_t)-1);
2584e86cc2d8c31ba30a4109cf4ba09d1e37e28aTimo Sirainen while ((line = i_stream_read_next_line(file->input)) != NULL) {
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen if (*line == '\0') {
4b41116563110d00330896a568eff1078c382827Timo Sirainen /* end of metadata */
4b41116563110d00330896a568eff1078c382827Timo Sirainen break;
4b41116563110d00330896a568eff1078c382827Timo Sirainen }
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen if (*line == DBOX_METADATA_EXT_REF) T_BEGIN {
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen if (!index_attachment_parse_extrefs(line+1, ext_refs_pool,
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen extrefs)) {
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen i_warning("%s: Ignoring corrupted extref: %s",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen file->cur_path, line);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen } T_END;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
b2c1349cf07410aefab0f5b17153af9e5cfcf48fTimo Sirainen i_stream_set_max_buffer_size(file->input, buf_size);
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (line == NULL) {
dbe64f3893616a4005c8946be75d2dc8f6823d72Timo Sirainen dbox_file_set_corrupted(file, "missing end-of-metadata line");
8a13b020f90e080570658b18c042e7e352c8b14fTimo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return 1;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen}
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenstatic bool
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenmdbox_purge_want_altpath(struct mdbox_purge_context *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct dbox_file *file, uint32_t map_uid)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen enum mdbox_msg_action action;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen void *value;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen if (dbox_file_is_in_alt(file) &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->storage->set->mdbox_purge_preserve_alt)
f3bb2fbe87425dc89a839908985af496f7f65702Timo Sirainen return TRUE;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen
a3ee5ce6ecc8e228ee69300fdd562d7ac8be89a7Timo Sirainen if (!ctx->have_altmoves)
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainen return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid));
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen action = POINTER_CAST_TO(value, enum mdbox_msg_action);
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen return action == MDBOX_MSG_ACTION_MOVE_TO_ALT;
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen}
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainenstatic int
bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainenmdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mdbox_map_file_msg *msg)
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen{
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen struct dbox_file_append_context *out_file_append;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen struct istream *input;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen struct ostream *output;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen enum mdbox_map_append_flags append_flags;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen uoff_t msg_size;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen off_t ret;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen int read_errno;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ctx->append_ctx == NULL)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen ctx->append_ctx = mdbox_map_append_begin(ctx->atomic);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen append_flags = !mdbox_purge_want_altpath(ctx, file, msg->map_uid) ? 0 :
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen DBOX_MAP_APPEND_FLAG_ALT;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen msg_size = file->msg_header_size + file->cur_physical_size;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen if (mdbox_map_append_next(ctx->append_ctx, file->cur_physical_size,
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen append_flags, &out_file_append, &output) < 0)
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen return -1;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen i_assert(file != out_file_append->file);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen input = i_stream_create_limit(file->input, msg_size);
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen ret = o_stream_send_istream(output, input);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen read_errno = input->stream_errno;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen i_stream_unref(&input);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (read_errno != 0) {
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen errno = read_errno;
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen mail_storage_set_critical(&file->storage->storage,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen "read(%s) failed: %m", file->cur_path);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen if (o_stream_nfinish(output) < 0) {
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mail_storage_set_critical(&file->storage->storage,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen "write(%s) failed: %m",
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen out_file_append->file->cur_path);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen return -1;
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen }
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen if (ret != (off_t)msg_size) {
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen i_assert(ret < (off_t)msg_size);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen i_assert(i_stream_is_eof(file->input));
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen dbox_file_set_corrupted(file, "truncated message at EOF");
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen return 0;
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen }
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen /* copy metadata */
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen if ((ret = mdbox_file_metadata_copy(file, output)) <= 0)
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen return ret;
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen mdbox_map_append_finish(ctx->append_ctx);
42507d758b053bb483de58fba55c73a9eb5d3fbaTimo Sirainen return 1;
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen}
42507d758b053bb483de58fba55c73a9eb5d3fbaTimo Sirainen
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainenstatic int
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainenmdbox_file_purge_check_refcounts(struct mdbox_purge_context *ctx,
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen const ARRAY_TYPE(mdbox_map_file_msg) *msgs_arr)
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen{
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen struct mdbox_map *map = ctx->storage->map;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen struct mdbox_map_mail_index_record rec;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen uint16_t refcount;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen const struct mdbox_map_file_msg *msgs;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen unsigned int i, count;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen int ret;
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen if (mdbox_map_atomic_lock(ctx->atomic) < 0)
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen return -1;
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen msgs = array_get(msgs_arr, &count);
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen for (i = 0; i < count; i++) {
7baab0b0b60df7ce9093d0881cd322dff1e79491Timo Sirainen if (msgs[i].refcount != 0)
7baab0b0b60df7ce9093d0881cd322dff1e79491Timo Sirainen continue;
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen ret = mdbox_map_lookup_full(map, msgs[i].map_uid, &rec,
6b0d8106ae51ffc6ce45636b34d2e21cbefca7fdTimo Sirainen &refcount);
6b0d8106ae51ffc6ce45636b34d2e21cbefca7fdTimo Sirainen if (ret <= 0) {
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen if (ret < 0)
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen return -1;
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen mdbox_map_set_corrupted(map,
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen "Purging unexpectedly lost map_uid=%u",
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen msgs[i].map_uid);
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen return -1;
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen }
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen if (refcount > 0)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return 0;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return 1;
0d86aa0d47f7393c669c084b34c0537b193688adTimo Sirainen}
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
7631f16156aca373004953fe6b01a7f343fb47e0Timo Sirainenstatic int
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainenmdbox_purge_attachments(struct mdbox_purge_context *ctx,
aa247243412a49f9bdebf7255e131dc6ece4ed46Timo Sirainen const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen{
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen struct dbox_storage *storage = &ctx->storage->storage;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen const struct mail_attachment_extref *extref;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen int ret = 0;
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen array_foreach(extrefs_arr, extref) {
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen if (index_attachment_delete(&storage->storage,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen storage->attachment_fs,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen extref->path) < 0)
c0225f7f6b43d34dc58c17d3304f0fd60ab89894Timo Sirainen ret = -1;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return ret;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen}
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenstatic int
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenmdbox_file_purge(struct mdbox_purge_context *ctx, struct dbox_file *file,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen uint32_t file_id)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen{
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct stat st;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ARRAY_TYPE(mdbox_map_file_msg) msgs_arr;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen const struct mdbox_map_file_msg *msgs;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ARRAY_TYPE(seq_range) expunged_map_uids;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ARRAY_TYPE(uint32_t) copied_map_uids;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ARRAY_TYPE(mail_attachment_extref) ext_refs;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen pool_t ext_refs_pool;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen unsigned int i, count;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen uoff_t offset;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen int ret;
9af6cc9ebc9986c1275ebdfa29c39e152af1557eTimo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen i_assert(ctx->atomic == NULL);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen i_assert(ctx->append_ctx == NULL);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen if ((ret = dbox_file_try_lock(file)) <= 0)
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen return ret;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen
225e82df5dd1e765f4e52b80c954558f00e5a7dfTimo Sirainen /* make sure the file still exists. another process may have already
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen deleted it. */
c7e14824e4e1ca9dc5d48d1eddc4a38d3041218fTimo Sirainen if (stat(file->cur_path, &st) < 0) {
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen dbox_file_unlock(file);
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen if (errno == ENOENT)
6564208826b0f46a00f010d1b5711d85944c3c88Timo Sirainen return 0;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mail_storage_set_critical(&file->storage->storage,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen "stat(%s) failed: %m", file->cur_path);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return -1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* get list of map UIDs that exist in this file (again has to be done
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen after locking) */
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen i_array_init(&msgs_arr, 128);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen if (mdbox_map_get_file_msgs(dstorage->map, file_id,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen &msgs_arr) < 0) {
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen array_free(&msgs_arr);
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen dbox_file_unlock(file);
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen return -1;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* sort messages by their offset */
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp);
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen ext_refs_pool = pool_alloconly_create("mdbox purge ext refs", 1024);
1eb17e61d3d38372674aa0c55caedb0185a985f5Timo Sirainen ctx->atomic = mdbox_map_atomic_begin(ctx->storage->map);
1eb17e61d3d38372674aa0c55caedb0185a985f5Timo Sirainen msgs = array_get(&msgs_arr, &count);
1eb17e61d3d38372674aa0c55caedb0185a985f5Timo Sirainen i_array_init(&ext_refs, 32);
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen i_array_init(&copied_map_uids, I_MIN(count, 1));
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen i_array_init(&expunged_map_uids, I_MIN(count, 1));
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen offset = file->file_header_size;
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen for (i = 0; i < count; i++) {
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen if ((ret = dbox_file_seek(file, offset)) <= 0)
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen break;
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen if (msgs[i].offset != offset) {
b0a901f1dbe9e05ac1c92a0974af6bce0274f31aTimo Sirainen /* map doesn't match file's actual contents */
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen dbox_file_set_corrupted(file,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen "purging found mismatched offsets "
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "(%"PRIuUOFF_T" vs %u, %u/%u)",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen offset, msgs[i].offset, i, count);
e4c90f0b88e40a8f92b8f5e1f1a3ea701e5c965cTimo Sirainen ret = 0;
defb12ecd360df672ffb2f4dbf4d1218a0a9549cTimo Sirainen break;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (msgs[i].refcount == 0) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* skip over expunged message */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen i_stream_seek(file->input, offset +
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen file->msg_header_size +
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen file->cur_physical_size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* skip metadata */
194603b35061fea1ee8d171a7104b6985c610966Timo Sirainen ret = mdbox_metadata_get_extrefs(file, ext_refs_pool,
194603b35061fea1ee8d171a7104b6985c610966Timo Sirainen &ext_refs);
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (ret <= 0)
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen break;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen seq_range_array_add(&expunged_map_uids,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen msgs[i].map_uid);
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen } else {
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen /* non-expunged message. write it to output file. */
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen i_stream_seek(file->input, offset);
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen ret = mdbox_purge_save_msg(ctx, file, &msgs[i]);
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (ret <= 0)
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen break;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen array_append(&copied_map_uids, &msgs[i].map_uid, 1);
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen offset = file->input->v_offset;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen if (offset != (uoff_t)st.st_size && ret > 0) {
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen /* file has more messages than what map tells us */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen dbox_file_set_corrupted(file,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen "more messages available than in map "
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen ret = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (ret > 0 && ctx->append_ctx != NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* flush writes before locking the map */
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (mdbox_map_append_flush(ctx->append_ctx) < 0)
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen ret = -1;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen }
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (ret <= 0)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen ret = -1;
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen else {
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen /* it's possible that one of the messages we purged was
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen just copied to another mailbox. the only way to prevent that
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen would be to keep map locked during the purge, but that could
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen keep it locked for too long. instead we'll check here if
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen there are such copies, and if there are cancel this file's
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen purge. */
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen ret = mdbox_file_purge_check_refcounts(ctx, &msgs_arr);
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen }
a3ee5ce6ecc8e228ee69300fdd562d7ac8be89a7Timo Sirainen array_free(&msgs_arr); msgs = NULL;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen if (ret <= 0) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* failed */
4321f6c969e7b8f6b243ff5bb6b8d297921676f6Timo Sirainen } else if (ctx->append_ctx == NULL) {
4321f6c969e7b8f6b243ff5bb6b8d297921676f6Timo Sirainen /* everything purged from this file */
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen ret = 1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* assign new file_id + offset to moved messages */
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen if (mdbox_map_append_move(ctx->append_ctx, &copied_map_uids,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen &expunged_map_uids) < 0 ||
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen mdbox_map_append_commit(ctx->append_ctx) < 0)
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen ret = -1;
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen else
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen ret = 1;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen }
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen if (ctx->append_ctx != NULL)
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen mdbox_map_append_free(&ctx->append_ctx);
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen (void)mdbox_map_atomic_finish(&ctx->atomic);
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen /* unlink only after unlocking map, so readers don't see it
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen temporarily vanished */
e050e5c9b1688765f1fdfce9b7141f7b614383fdTimo Sirainen if (ret > 0) {
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen (void)dbox_file_unlink(file);
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen if (mdbox_map_remove_file_id(ctx->storage->map, file_id) < 0)
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen ret = -1;
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen } else {
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen dbox_file_unlock(file);
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen }
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen array_free(&copied_map_uids);
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen array_free(&expunged_map_uids);
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen (void)mdbox_purge_attachments(ctx, &ext_refs);
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen array_free(&ext_refs);
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen pool_unref(&ext_refs_pool);
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen return ret;
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen}
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainenvoid mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen{
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)mail->box;
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen ARRAY_TYPE(uint32_t) *dest;
de58be41126e5d68008d2ea706d62ccdc1f29337Timo Sirainen uint32_t map_uid;
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen /* we'll assume here that alt flag won't be changed multiple times
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen for the same mail. it shouldn't happen with current code, and
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen checking for it would just slow down the code.
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen so the way it works currently is just that map_uids are added to
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen an array, which is later sorted and processed further. note that
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen it's still possible that the same map_uid exists in the array
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen multiple times. */
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen if (mdbox_mail_lookup(mbox, mbox->box.view, mail->seq, &map_uid) < 0)
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen return;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen dest = move_to_alt ? &mbox->storage->move_to_alt_map_uids :
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen &mbox->storage->move_from_alt_map_uids;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (!array_is_created(dest))
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen i_array_init(dest, 256);
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen array_append(dest, &map_uid, 1);
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen}
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainenstatic struct mdbox_purge_context *
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmdbox_purge_alloc(struct mdbox_storage *storage)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mdbox_purge_context *ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen pool_t pool;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen pool = pool_alloconly_create("mdbox purge context", 1024*32);
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen ctx = p_new(pool, struct mdbox_purge_context, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ctx->pool = pool;
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen ctx->storage = storage;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen ctx->lowest_primary_file_id = (uint32_t)-1;
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen i_array_init(&ctx->primary_file_ids, 64);
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen i_array_init(&ctx->purge_file_ids, 64);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen hash_table_create_direct(&ctx->altmoves, pool, 0);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return ctx;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen}
ebe6df72f1309135f02b6a4d2aef1e81a073f91cTimo Sirainen
910fa4e4204a73d3d24c03f3059dd24e727ca057Timo Sirainenstatic void mdbox_purge_free(struct mdbox_purge_context **_ctx)
7631f16156aca373004953fe6b01a7f343fb47e0Timo Sirainen{
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen struct mdbox_purge_context *ctx = *_ctx;
b83deefd2cf1e293373673eefb4d5cf60907978cTimo Sirainen
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen *_ctx = NULL;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen hash_table_destroy(&ctx->altmoves);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen array_free(&ctx->primary_file_ids);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen array_free(&ctx->purge_file_ids);
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen pool_unref(&ctx->pool);
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen}
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen
3c493c276f599d9b9cd10764876d648003046954Timo Sirainenstatic int mdbox_purge_get_primary_files(struct mdbox_purge_context *ctx)
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen{
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen struct mdbox_storage *dstorage = ctx->storage;
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen struct mail_storage *storage = &dstorage->storage.storage;
b3b4f3875850099c9292ad74d08bb385c3988f8fTimo Sirainen DIR *dir;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct dirent *d;
string_t *path;
unsigned int file_id, dir_len;
int ret = 0;
if (!array_is_created(&dstorage->move_to_alt_map_uids) &&
!array_is_created(&dstorage->move_from_alt_map_uids)) {
/* we don't need to do alt moving, don't bother getting list
of primary files */
return 0;
}
dir = opendir(dstorage->storage_dir);
if (dir == NULL) {
if (errno == ENOENT) {
/* no storage directory at all yet */
return 0;
}
mail_storage_set_critical(storage,
"opendir(%s) failed: %m", dstorage->storage_dir);
return -1;
}
path = t_str_new(256);
str_append(path, dstorage->storage_dir);
str_append_c(path, '/');
dir_len = str_len(path);
for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
continue;
if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
&file_id) < 0)
continue;
str_truncate(path, dir_len);
str_append(path, d->d_name);
seq_range_array_add(&ctx->primary_file_ids, file_id);
}
if (array_count(&ctx->primary_file_ids) > 0) {
const struct seq_range *range =
array_idx(&ctx->primary_file_ids, 0);
ctx->lowest_primary_file_id = range[0].seq1;
}
if (errno != 0) {
mail_storage_set_critical(storage,
"readdir(%s) failed: %m", dstorage->storage_dir);
ret = -1;
}
if (closedir(dir) < 0) {
mail_storage_set_critical(storage,
"closedir(%s) failed: %m", dstorage->storage_dir);
ret = -1;
}
return ret;
}
static int uint32_t_cmp(const uint32_t *u1, const uint32_t *u2)
{
if (*u1 < *u2)
return -1;
if (*u1 > *u2)
return 1;
return 0;
}
static int mdbox_altmove_add_files(struct mdbox_purge_context *ctx)
{
struct mdbox_storage *dstorage = ctx->storage;
const uint32_t *map_uids;
unsigned int i, count, alt_refcount = 0;
struct mdbox_map_mail_index_record cur_rec;
enum mdbox_msg_action action;
uint32_t cur_map_uid;
uint16_t cur_refcount = 0;
uoff_t offset;
int ret = 0;
/* first add move-to-alt actions */
if (array_is_created(&dstorage->move_to_alt_map_uids)) {
array_sort(&dstorage->move_to_alt_map_uids, uint32_t_cmp);
map_uids = array_get(&dstorage->move_to_alt_map_uids, &count);
} else {
map_uids = NULL;
count = 0;
}
cur_map_uid = 0;
for (i = 0; i < count; i++) {
if (cur_map_uid != map_uids[i]) {
cur_map_uid = map_uids[i];
if (mdbox_map_lookup_full(dstorage->map, cur_map_uid,
&cur_rec, &cur_refcount) < 0) {
cur_refcount = (uint16_t)-1;
ret = -1;
}
alt_refcount = 1;
} else {
alt_refcount++;
}
if (alt_refcount == cur_refcount &&
seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) {
/* all instances marked as moved to alt storage */
action = MDBOX_MSG_ACTION_MOVE_TO_ALT;
hash_table_insert(ctx->altmoves,
POINTER_CAST(cur_map_uid),
POINTER_CAST(action));
seq_range_array_add(&ctx->purge_file_ids,
cur_rec.file_id);
}
}
/* next add move-from-alt actions. they override move-to-alt actions
in case there happen to be any conflicts (shouldn't). only a single
move-from-alt record is needed to do the move. */
if (array_is_created(&dstorage->move_from_alt_map_uids))
map_uids = array_get(&dstorage->move_from_alt_map_uids, &count);
else {
map_uids = NULL;
count = 0;
}
cur_map_uid = 0;
for (i = 0; i < count; i++) {
if (cur_map_uid == map_uids[i])
continue;
cur_map_uid = map_uids[i];
if (mdbox_map_lookup(dstorage->map, cur_map_uid,
&cur_rec.file_id, &offset) < 0) {
ret = -1;
continue;
}
if (seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) {
/* already in primary storage */
continue;
}
action = MDBOX_MSG_ACTION_MOVE_FROM_ALT;
hash_table_insert(ctx->altmoves, POINTER_CAST(cur_map_uid),
POINTER_CAST(action));
seq_range_array_add(&ctx->purge_file_ids, cur_rec.file_id);
}
ctx->have_altmoves = hash_table_count(ctx->altmoves) > 0;
return ret;
}
int mdbox_purge(struct mail_storage *_storage)
{
struct mdbox_storage *storage = (struct mdbox_storage *)_storage;
struct mdbox_purge_context *ctx;
struct dbox_file *file;
struct seq_range_iter iter;
unsigned int i = 0;
uint32_t file_id;
bool deleted;
int ret;
ctx = mdbox_purge_alloc(storage);
ret = mdbox_map_get_zero_ref_files(storage->map, &ctx->purge_file_ids);
if (storage->alt_storage_dir != NULL) {
if (mdbox_purge_get_primary_files(ctx) < 0)
ret = -1;
else {
/* add files that can be altmoved */
if (mdbox_altmove_add_files(ctx) < 0)
ret = -1;
}
}
seq_range_array_iter_init(&iter, &ctx->purge_file_ids); i = 0;
while (ret == 0 &&
seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN {
file = mdbox_file_init(storage, file_id);
if (dbox_file_open(file, &deleted) > 0 && !deleted) {
if (mdbox_file_purge(ctx, file, file_id) < 0)
ret = -1;
} else {
if (mdbox_map_remove_file_id(storage->map, file_id) < 0)
ret = -1;
}
dbox_file_unref(&file);
} T_END;
mdbox_purge_free(&ctx);
if (storage->corrupted) {
/* purging found corrupted files */
(void)mdbox_storage_rebuild(storage);
ret = -1;
}
return ret;
}