mail-index-transaction-export.c revision 9e6d83a3ef6abb393eeebca423cfd0d8cb08d430
89a126810703c666309310d0f3189e9834d70b5bTimo Sirainen/* Copyright (c) 2003-2016 Dovecot authors, see the included COPYING file */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen#include "array.h"
16f816d3f3c32ae3351834253f52ddd0212bcbf3Timo Sirainen#include "mail-index-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-modseq.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "mail-transaction-log-private.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "mail-index-transaction-private.h"
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstruct mail_index_export_context {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_transaction *trans;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen struct mail_transaction_log_append_ctx *append_ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen};
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic void
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenlog_append_buffer(struct mail_index_export_context *ctx,
dd0ba1bab2c1b89c7e063fa45d156fa72b8260d5Timo Sirainen const buffer_t *buf, enum mail_transaction_type type)
dd0ba1bab2c1b89c7e063fa45d156fa72b8260d5Timo Sirainen{
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen mail_transaction_log_append_add(ctx->append_ctx, type,
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen buf->data, buf->used);
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen}
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainenstatic void log_append_flag_updates(struct mail_index_export_context *ctx,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_transaction *t)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ARRAY(struct mail_transaction_flag_update) log_updates;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const struct mail_index_flag_update *updates;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_transaction_flag_update *log_update;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen unsigned int i, count;
956f7778e413d3184d69e7b96e4a6b3cd5570bcdTimo Sirainen
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen updates = array_get(&t->updates, &count);
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen if (count == 0)
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen return;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen i_array_init(&log_updates, count);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen for (i = 0; i < count; i++) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_update = array_append_space(&log_updates);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_update->uid1 = updates[i].uid1;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_update->uid2 = updates[i].uid2;
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen log_update->add_flags = updates[i].add_flags & 0xff;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_update->remove_flags = updates[i].remove_flags & 0xff;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen log_update->modseq_inc_flag = 1;
e1f05b193ac1edd3267294e9501e8063aa0f791aTimo Sirainen }
e1f05b193ac1edd3267294e9501e8063aa0f791aTimo Sirainen log_append_buffer(ctx, log_updates.arr.buffer,
e1f05b193ac1edd3267294e9501e8063aa0f791aTimo Sirainen MAIL_TRANSACTION_FLAG_UPDATE);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen array_free(&log_updates);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic const buffer_t *
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenlog_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_t *buf;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const unsigned char *data, *mask;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_transaction_header_update u;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen uint16_t offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int state = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen memset(&u, 0, sizeof(u));
91dca97b367c54a139c268b56a0c67f564bd9197Timo Sirainen
ee26329cb5cc679b5645e4933d529f86accb976aTimo Sirainen data = prepend ? t->pre_hdr_change : t->post_hdr_change;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 256);
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (offset < sizeof(t->pre_hdr_change) && mask[offset] != 0) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (state == 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen u.offset = offset;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen state++;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen }
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen } else {
1a5fcc972dbadfe7959011b8ad422707e2dfc19fTimo Sirainen if (state > 0) {
1a5fcc972dbadfe7959011b8ad422707e2dfc19fTimo Sirainen u.size = offset - u.offset;
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen buffer_append(buf, &u, sizeof(u));
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen buffer_append(buf, data + u.offset, u.size);
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen state = 0;
cfbf891f065b18602703ed6fa2af1a541d4d0b04Timo Sirainen }
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen }
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen }
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen return buf;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen}
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainenstatic unsigned int
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainenext_hdr_update_get_size(const struct mail_index_transaction_ext_hdr_update *hu)
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen{
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen unsigned int i;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen for (i = hu->alloc_size; i > 0; i--) {
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen if (hu->mask[i-1] != 0)
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen return i;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen }
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen return 0;
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainenstatic void log_append_ext_intro(struct mail_index_export_context *ctx,
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen uint32_t ext_id, uint32_t reset_id,
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen unsigned int *hdr_size_r)
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen{
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen struct mail_index_transaction *t = ctx->trans;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen const struct mail_index_registered_ext *rext;
ca316aeb7648d3f1bcf45231f73ddeb1b67a6961Timo Sirainen const struct mail_index_ext *ext;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen struct mail_transaction_ext_intro *intro, *resizes;
6f73af3a3a6ee900c7e736874587968d76a20bc0Timo Sirainen buffer_t *buf;
50c4a9739b55370b1d3950d7b3ec2f7cd2ed5f49Timo Sirainen uint32_t idx;
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen unsigned int count;
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen i_assert(ext_id != (uint32_t)-1);
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (t->reset ||
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* new extension */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen idx = (uint32_t)-1;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen }
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen rext = array_idx(&t->view->index->extensions, ext_id);
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen if (!array_is_created(&t->ext_resizes)) {
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen resizes = NULL;
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen count = 0;
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen } else {
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen resizes = array_get_modifiable(&t->ext_resizes, &count);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen }
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 128);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen if (ext_id < count && resizes[ext_id].name_size != 0) {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen /* we're resizing the extension. use the resize struct. */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro = &resizes[ext_id];
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen if (idx != (uint32_t)-1) {
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen intro->ext_id = idx;
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen intro->name_size = 0;
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen } else {
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen intro->ext_id = (uint32_t)-1;
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen intro->name_size = strlen(rext->name);
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen }
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen buffer_append(buf, intro, sizeof(*intro));
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen } else {
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen /* generate a new intro structure */
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen intro = buffer_append_space_unsafe(buf, sizeof(*intro));
d24daa83c25063ef12b524c9ffcc9ecca34dadb9Timo Sirainen intro->ext_id = idx;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro->record_size = rext->record_size;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro->record_align = rext->record_align;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen if (idx == (uint32_t)-1) {
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen intro->hdr_size = rext->hdr_size;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen intro->name_size = strlen(rext->name);
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen } else {
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen ext = array_idx(&t->view->index->map->extensions, idx);
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen intro->hdr_size = ext->hdr_size;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro->name_size = 0;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen }
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen /* handle increasing header size automatically */
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen if (array_is_created(&t->ext_hdr_updates) &&
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen ext_id < array_count(&t->ext_hdr_updates)) {
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen const struct mail_index_transaction_ext_hdr_update *hu;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen unsigned int hdr_update_size;
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen hu = array_idx(&t->ext_hdr_updates, ext_id);
31050c3df6cbe403e8ced8ef11b5c4e12124d354Timo Sirainen hdr_update_size = ext_hdr_update_get_size(hu);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen if (intro->hdr_size < hdr_update_size)
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro->hdr_size = hdr_update_size;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen }
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen }
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen i_assert(intro->record_size != 0 || intro->hdr_size != 0);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen if (reset_id != 0) {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen /* we're going to reset this extension in this transaction */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro->reset_id = reset_id;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen } else if (idx != (uint32_t)-1) {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen /* use the existing reset_id */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen const struct mail_index_ext *map_ext =
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen array_idx(&t->view->index->map->extensions, idx);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen intro->reset_id = map_ext->reset_id;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen } else {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen /* new extension, reset_id defaults to 0 */
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen }
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen buffer_append(buf, rext->name, intro->name_size);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen if ((buf->used % 4) != 0)
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen buffer_append_zero(buf, 4 - (buf->used % 4));
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen if (ctx->append_ctx->new_highest_modseq == 0 &&
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen /* modseq tracking started */
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen ctx->append_ctx->new_highest_modseq = 1;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen }
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO);
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen *hdr_size_r = intro->hdr_size;
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen}
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainenstatic void
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainenlog_append_ext_hdr_update(struct mail_index_export_context *ctx,
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen const struct mail_index_transaction_ext_hdr_update *hdr,
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen unsigned int ext_hdr_size)
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen{
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen buffer_t *buf;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen const unsigned char *data, *mask;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen struct mail_transaction_ext_hdr_update u;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen struct mail_transaction_ext_hdr_update32 u32;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen size_t offset;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen bool started = FALSE, use_32 = hdr->alloc_size >= 65536;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen memset(&u, 0, sizeof(u));
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen memset(&u32, 0, sizeof(u32));
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen
dc0474dc9d5652d76cb41f439844dd80c8b96642Timo Sirainen data = hdr->data;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen mask = hdr->mask;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen buf = buffer_create_dynamic(pool_datastack_create(), 256);
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen for (offset = 0; offset <= hdr->alloc_size; offset++) {
cfbf891f065b18602703ed6fa2af1a541d4d0b04Timo Sirainen if (offset < hdr->alloc_size && mask[offset] != 0) {
a2ec607689dc88112bf08785960e441153f35d57Timo Sirainen if (!started) {
a2ec607689dc88112bf08785960e441153f35d57Timo Sirainen u32.offset = offset;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen started = TRUE;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen }
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen } else {
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen if (started) {
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen u32.size = offset - u32.offset;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen if (use_32)
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen buffer_append(buf, &u32, sizeof(u32));
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen else {
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen u.offset = u32.offset;
b159b7fb9740b6e37238016d8395a351de498d50Timo Sirainen u.size = u32.size;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen buffer_append(buf, &u, sizeof(u));
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen }
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen i_assert(u32.offset + u32.size <= ext_hdr_size);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_append(buf, data + u32.offset, u32.size);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen started = FALSE;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (buf->used % 4 != 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append_zero(buf, 4 - buf->used % 4);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 :
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen MAIL_TRANSACTION_EXT_HDR_UPDATE);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstatic void
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenmail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_transaction *t = ctx->trans;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const struct mail_transaction_ext_intro *resize;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const struct mail_index_transaction_ext_hdr_update *hdrs;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen struct mail_transaction_ext_reset ext_reset;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen unsigned int resize_count, ext_count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int hdrs_count, reset_id_count, reset_count, hdr_size;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen uint32_t ext_id, reset_id;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const struct mail_transaction_ext_reset *reset;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const uint32_t *reset_ids;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_t reset_buf;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (!array_is_created(&t->ext_resizes)) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen resize = NULL;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen resize_count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen resize = array_get(&t->ext_resizes, &resize_count);
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen if (ext_count < resize_count)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext_count = resize_count;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (!array_is_created(&t->ext_reset_ids)) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen reset_ids = NULL;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen reset_id_count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen } else {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (!array_is_created(&t->ext_resets)) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen reset = NULL;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen reset_count = 0;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen } else {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen reset = array_get(&t->ext_resets, &reset_count);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (ext_count < reset_count)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ext_count = reset_count;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (!array_is_created(&t->ext_hdr_updates)) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen hdrs = NULL;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen hdrs_count = 0;
faef57a36800fe952a5794328467f846d5480ae6Timo Sirainen } else {
faef57a36800fe952a5794328467f846d5480ae6Timo Sirainen hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (ext_count < hdrs_count)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext_count = hdrs_count;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen memset(&ext_reset, 0, sizeof(ext_reset));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_create_from_data(&reset_buf, &ext_reset, sizeof(ext_reset));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_set_used_size(&reset_buf, sizeof(ext_reset));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen for (ext_id = 0; ext_id < ext_count; ext_id++) {
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen if (ext_id < reset_count)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ext_reset = reset[ext_id];
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen else
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ext_reset.new_reset_id = 0;
faef57a36800fe952a5794328467f846d5480ae6Timo Sirainen if ((ext_id < resize_count && resize[ext_id].name_size > 0) ||
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ext_reset.new_reset_id != 0 ||
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (ext_reset.new_reset_id != 0) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen /* we're going to reset this extension
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen immediately after the intro */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen reset_id = 0;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen } else {
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen reset_id = ext_id < reset_id_count ?
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen reset_ids[ext_id] : 0;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size);
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen } else {
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen hdr_size = 0;
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen }
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen if (ext_reset.new_reset_id != 0) {
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen i_assert(ext_id < reset_id_count &&
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen ext_reset.new_reset_id == reset_ids[ext_id]);
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen log_append_buffer(ctx, &reset_buf,
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen MAIL_TRANSACTION_EXT_RESET);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen T_BEGIN {
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen log_append_ext_hdr_update(ctx, &hdrs[ext_id],
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen hdr_size);
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen } T_END;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen }
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen }
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen}
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainenstatic void log_append_ext_recs(struct mail_index_export_context *ctx,
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen const ARRAY_TYPE(seq_array_array) *arr,
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen enum mail_transaction_type type)
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen{
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen struct mail_index_transaction *t = ctx->trans;
1f6653c23f792401623734d783ede1653ba65436Timo Sirainen const ARRAY_TYPE(seq_array) *updates;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen const uint32_t *reset_ids;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen unsigned int ext_id, count, reset_id_count, hdr_size;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen uint32_t reset_id;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen if (!array_is_created(&t->ext_reset_ids)) {
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen reset_ids = NULL;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen reset_id_count = 0;
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen } else {
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen reset_ids = array_get_modifiable(&t->ext_reset_ids,
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen &reset_id_count);
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen }
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen updates = array_get(arr, &count);
2623ed55e5bc547363abc7bc9de6b4ad2c53e531Timo Sirainen for (ext_id = 0; ext_id < count; ext_id++) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (!array_is_created(&updates[ext_id]))
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen continue;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_append_buffer(ctx, updates[ext_id].arr.buffer, type);
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen }
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstatic void
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenlog_append_keyword_update(struct mail_index_export_context *ctx,
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen buffer_t *tmp_buf, enum modify_type modify_type,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const char *keyword, const buffer_t *uid_buffer)
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen{
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen struct mail_transaction_keyword_update kt_hdr;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen i_assert(uid_buffer->used > 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen memset(&kt_hdr, 0, sizeof(kt_hdr));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen kt_hdr.modify_type = modify_type;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen kt_hdr.name_size = strlen(keyword);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_set_used_size(tmp_buf, 0);
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_append(tmp_buf, keyword, kt_hdr.name_size);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if ((tmp_buf->used % 4) != 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append(tmp_buf, uid_buffer->data, uid_buffer->used);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE);
0a51697f82fbd45a511710479e99efd42dc18453Timo Sirainen}
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainenstatic enum mail_index_fsync_mask
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainenlog_append_keyword_updates(struct mail_index_export_context *ctx)
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen{
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen const struct mail_index_transaction_keyword_update *updates;
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen const char *const *keywords;
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen buffer_t *tmp_buf;
0a51697f82fbd45a511710479e99efd42dc18453Timo Sirainen enum mail_index_fsync_mask change_mask = 0;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen unsigned int i, count, keywords_count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen tmp_buf = buffer_create_dynamic(pool_datastack_create(), 64);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen &keywords_count);
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen i_assert(count <= keywords_count);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen for (i = 0; i < count; i++) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (array_is_created(&updates[i].add_seq) &&
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen array_count(&updates[i].add_seq) > 0) {
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_append_keyword_update(ctx, tmp_buf,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen MODIFY_ADD, keywords[i],
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen updates[i].add_seq.arr.buffer);
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen }
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen if (array_is_created(&updates[i].remove_seq) &&
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen array_count(&updates[i].remove_seq) > 0) {
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS;
6cd02a9525acb9897a65b05bfeee9b8d283e0f74Timo Sirainen log_append_keyword_update(ctx, tmp_buf,
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen MODIFY_REMOVE, keywords[i],
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen updates[i].remove_seq.arr.buffer);
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen }
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return change_mask;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenvoid mail_index_transaction_export(struct mail_index_transaction *t,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_transaction_log_append_ctx *append_ctx)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen static uint8_t null4[4] = { 0, 0, 0, 0 };
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen enum mail_index_fsync_mask change_mask = 0;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen struct mail_index_export_context ctx;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen memset(&ctx, 0, sizeof(ctx));
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ctx.trans = t;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ctx.append_ctx = append_ctx;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
cfbb3ff45d6bd27e6442e332fa69c43c984ca651Timo Sirainen if (t->index_undeleted) {
cfbb3ff45d6bd27e6442e332fa69c43c984ca651Timo Sirainen i_assert(!t->index_deleted);
cfbb3ff45d6bd27e6442e332fa69c43c984ca651Timo Sirainen mail_transaction_log_append_add(ctx.append_ctx,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen MAIL_TRANSACTION_INDEX_UNDELETED, &null4, 4);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen /* send all extension introductions and resizes before appends
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen to avoid resize overhead as much as possible */
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen mail_transaction_log_append_ext_intros(&ctx);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (t->pre_hdr_changed) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE),
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen MAIL_TRANSACTION_HEADER_UPDATE);
dbd9604da561399cc6255289d5b6f6f662ab2d00Timo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (t->attribute_updates != NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen buffer_append_c(t->attribute_updates, '\0');
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen /* need to have 32bit alignment */
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (t->attribute_updates->used % 4 != 0) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen buffer_append_zero(t->attribute_updates,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen 4 - t->attribute_updates->used % 4);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* append the timestamp and value lengths */
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen buffer_append(t->attribute_updates,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen t->attribute_updates_suffix->data,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen t->attribute_updates_suffix->used);
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen i_assert(t->attribute_updates->used % 4 == 0);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen log_append_buffer(&ctx, t->attribute_updates,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen MAIL_TRANSACTION_ATTRIBUTE_UPDATE);
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (array_is_created(&t->appends)) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen change_mask |= MAIL_INDEX_FSYNC_MASK_APPENDS;
a2ce2eb4c266e2854fd34416ea5cfbe05dfd3971Timo Sirainen log_append_buffer(&ctx, t->appends.arr.buffer,
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen MAIL_TRANSACTION_APPEND);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f79a807865222dc8d5afd21667d2ace67f6c831bTimo Sirainen if (array_is_created(&t->updates)) {
f79a807865222dc8d5afd21667d2ace67f6c831bTimo Sirainen change_mask |= MAIL_INDEX_FSYNC_MASK_FLAGS;
0e7a1bd9b7b39e57a22dbd4ba12df9b9603e6391Timo Sirainen log_append_flag_updates(&ctx, t);
f79a807865222dc8d5afd21667d2ace67f6c831bTimo Sirainen }
f79a807865222dc8d5afd21667d2ace67f6c831bTimo Sirainen
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen if (array_is_created(&t->ext_rec_updates)) {
b30499429feee4f16f1e09c7047101dc7ff38304Timo Sirainen log_append_ext_recs(&ctx, &t->ext_rec_updates,
f79a807865222dc8d5afd21667d2ace67f6c831bTimo Sirainen MAIL_TRANSACTION_EXT_REC_UPDATE);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen if (array_is_created(&t->ext_rec_atomics)) {
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen log_append_ext_recs(&ctx, &t->ext_rec_atomics,
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen MAIL_TRANSACTION_EXT_ATOMIC_INC);
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen }
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (array_is_created(&t->keyword_updates))
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen change_mask |= log_append_keyword_updates(&ctx);
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen /* keep modseq updates almost last */
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (array_is_created(&t->modseq_updates)) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen log_append_buffer(&ctx, t->modseq_updates.arr.buffer,
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen MAIL_TRANSACTION_MODSEQ_UPDATE);
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen if (array_is_created(&t->expunges)) {
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen /* non-external expunges are only requests, ignore them when
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen checking fsync_mask */
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen change_mask |= MAIL_INDEX_FSYNC_MASK_EXPUNGES;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen log_append_buffer(&ctx, t->expunges.arr.buffer,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen MAIL_TRANSACTION_EXPUNGE_GUID);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen if (t->post_hdr_changed) {
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen MAIL_TRANSACTION_HEADER_UPDATE);
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen }
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (t->index_deleted) {
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen i_assert(!t->index_undeleted);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen mail_transaction_log_append_add(ctx.append_ctx,
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen MAIL_TRANSACTION_INDEX_DELETED,
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen &null4, 4);
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen }
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen append_ctx->index_sync_transaction = t->sync_transaction;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen append_ctx->tail_offset_changed = t->tail_offset_changed;
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen append_ctx->want_fsync =
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen (t->view->index->fsync_mask & change_mask) != 0 ||
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainenstatic unsigned int
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainencount_modseq_incs_with(struct mail_index_transaction *t,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen ARRAY_TYPE(seq_range) *tmp_seqs,
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen const ARRAY_TYPE(seq_range) *orig_seqs)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!array_is_created(orig_seqs))
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return 0;
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen array_clear(tmp_seqs);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen array_append_array(tmp_seqs, orig_seqs);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen mail_index_transaction_seq_range_to_uid(t, tmp_seqs);
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen return array_count(tmp_seqs) > 0 ? 1 : 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen}
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenstatic unsigned int
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainenmail_index_transaction_keywords_count_modseq_incs(struct mail_index_transaction *t)
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen{
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen const struct mail_index_transaction_keyword_update *update;
a817fdcc43aedf423e2134091d5f83f91d64bcc9Timo Sirainen ARRAY_TYPE(seq_range) tmp_seqs;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int count = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen i_array_init(&tmp_seqs, 64);
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen array_foreach_modifiable(&t->keyword_updates, update) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen count += count_modseq_incs_with(t, &tmp_seqs, &update->add_seq);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen count += count_modseq_incs_with(t, &tmp_seqs, &update->remove_seq);
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen array_free(&tmp_seqs);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen return count;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen}
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainenuint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction *t)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen{
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen struct mail_transaction_log_file *file = t->view->index->log->head;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen uint64_t new_highest_modseq = file->sync_highest_modseq;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen i_assert(file->locked);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (new_highest_modseq == 0) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* highest-modseq tracking isn't enabled in this transaction
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen log file. This shouldn't happen with logs created since
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen v2.2.26+, because initial_modseq is always set. We don't
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen also bother checking if this transaction itself enables the
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen highest-modseq tracking, because it's always done as a
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen standalone transaction in mail_index_modseq_enable(),
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen which doesn't care about this function. */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen i_warning("%s: Requested highest-modseq for transaction, "
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen "but modseq tracking isn't enabled for the file "
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen "(this shouldn't happen)", file->filepath);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen return 0;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* finish everything that can affect highest-modseq */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen mail_index_transaction_finish_so_far(t);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* NOTE: keep in sync with mail_transaction_update_modseq() */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (array_is_created(&t->appends) && array_count(&t->appends) > 0) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* sorting may change the order of keyword_updates, */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen new_highest_modseq++;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (array_is_created(&t->updates) && array_count(&t->updates) > 0)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen new_highest_modseq++;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (array_is_created(&t->keyword_updates)) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen new_highest_modseq +=
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen mail_index_transaction_keywords_count_modseq_incs(t);
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen }
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (t->attribute_updates != NULL)
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen new_highest_modseq++;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen /* NOTE: the order of modseq_updates and everything following it
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen must match mail_index_transaction_export(). */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen if (array_is_created(&t->modseq_updates)) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen const struct mail_transaction_modseq_update *mu;
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen /* mail_index_update_highest_modseq() is handled here also,
e37fbcda56ab154557e84f990012502be53aa6c6Timo Sirainen as a special case of uid==0. */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen array_foreach(&t->modseq_updates, mu) {
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen uint64_t modseq = ((uint64_t)mu->modseq_high32 << 32) |
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen mu->modseq_low32;
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (new_highest_modseq < modseq)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen new_highest_modseq = modseq;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen }
72cbf33ae81fde08384d30c779ff540752d9256cTimo Sirainen if (array_is_created(&t->expunges) && array_count(&t->expunges) > 0 &&
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen (t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen new_highest_modseq++;
e9f2d9104d395bcf54be3f8ba8d9f63aecf0bcbeTimo Sirainen return new_highest_modseq;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen}
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen