mail-transaction-log-view.c revision 096953143c4032bad154637f687551856f7946cb
45312f52ff3a3d4c137447be4c7556500c2f8bf2Timo Sirainen/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen /* a list of log files we've referenced. we have to keep this list
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen explicitly because more files may be added into the linked list
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen at any time. */
213b139965e8bde6c8aff02ffd9fd39a74c887a9Timo Sirainen ARRAY_DEFINE(file_refs, struct mail_transaction_log_file *);
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen struct mail_transaction_log_file *cur, *head, *tail;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_transaction_log_view_open(struct mail_transaction_log *log)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen view = i_new(struct mail_transaction_log_view, 1);
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen array_append(&view->file_refs, &view->head, 1);
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainenmail_transaction_log_view_unref_all(struct mail_transaction_log_view *view)
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen struct mail_transaction_log_file *const *files;
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen unsigned int i, count;
bbce20cb4e5739e9a06058cf8ee1f38a7f6884f6Timo Sirainen for (i = 0; i < count; i++)
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainenvoid mail_transaction_log_view_close(struct mail_transaction_log_view **_view)
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen struct mail_transaction_log_view *view = *_view;
93b29720c5141f787bd1861796867e4595c9d084Timo Sirainen for (p = &view->log->views; *p != NULL; p = &(*p)->next) {
93b29720c5141f787bd1861796867e4595c9d084Timo Sirainenvoid mail_transaction_log_views_close(struct mail_transaction_log *log)
93b29720c5141f787bd1861796867e4595c9d084Timo Sirainen for (view = log->views; view != NULL; view = view->next)
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainenint mail_transaction_log_view_set(struct mail_transaction_log_view *view,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen uint32_t min_file_seq, uoff_t min_file_offset,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen uint32_t max_file_seq, uoff_t max_file_offset,
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen struct mail_transaction_log_file *file, *const *files;
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen unsigned int i;
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen /* transaction log is closed already. this log view shouldn't
a050ca9def13949dbaa67bd6574a41c4f397ae26Timo Sirainen be used anymore. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* index file doesn't exist yet. this transaction log should
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen start from the beginning */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (view->log->files->hdr.prev_file_seq != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* but it doesn't */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen min_file_seq = view->log->files->hdr.file_seq;
45e62043058738e294f89504c319d852e25943ccTimo Sirainen if (min_file_seq == view->log->files->hdr.prev_file_seq &&
45e62043058738e294f89504c319d852e25943ccTimo Sirainen min_file_offset == view->log->files->hdr.prev_file_offset) {
9df8c9225140d9d1df5ddf4c6c9da61662ae6c44Timo Sirainen /* we can skip this */
45e62043058738e294f89504c319d852e25943ccTimo Sirainen min_file_seq = view->log->files->hdr.file_seq;
9df8c9225140d9d1df5ddf4c6c9da61662ae6c44Timo Sirainen /* empty view */
95a284736b8b11319a3f575ba249ba2eb7dbac1bTimo Sirainen if (min_file_seq == max_file_seq && min_file_offset > max_file_offset) {
95a284736b8b11319a3f575ba249ba2eb7dbac1bTimo Sirainen /* log file offset is probably corrupted in the index file. */
95a284736b8b11319a3f575ba249ba2eb7dbac1bTimo Sirainen min_file_seq, min_file_offset, max_file_offset);
be18b5067f9f787179f04c49168060165ed6be08Timo Sirainen min_file_offset < view->log->files->hdr.hdr_size) {
a4d209d480d453566d331e870b8d0c99af7716c8Timo Sirainen /* log file offset is probably corrupted in the index file. */
a4d209d480d453566d331e870b8d0c99af7716c8Timo Sirainen ") < hdr_size (%u)",
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen for (seq = min_file_seq; seq <= max_file_seq; seq++) {
fc7b17677ac1a5fa3f7fe13d5ef7dcfea8d9b4a1Timo Sirainen if (file == NULL || file->hdr.file_seq != seq) {
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen /* see if we could find the missing file. if we know
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen the max. file sequence, make sure NFS attribute
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen cache gets flushed if necessary. */
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen bool nfs_flush = max_file_seq != (uint32_t)-1;
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen ret = mail_transaction_log_find_file(view->log, seq,
fc7b17677ac1a5fa3f7fe13d5ef7dcfea8d9b4a1Timo Sirainen /* not found / corrupted */
fc7b17677ac1a5fa3f7fe13d5ef7dcfea8d9b4a1Timo Sirainen if (file == NULL || file->hdr.file_seq != seq) {
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen if (file == NULL && max_file_seq == (uint32_t)-1 &&
e86d0d34fe365da4c7ca4312d575bfcbf3a01c0eTimo Sirainen /* we just wanted to sync everything */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* if any of the found files reset the index,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen ignore any missing files up to it */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* missing files in the middle */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* we can ignore the missing file */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* beginning of the file */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* we don't actually want to show anything */
c21c33a8c98972c45349066fc76ac9e2c05013c1Timo Sirainen i_assert(min_file_offset >= view->tail->hdr.hdr_size);
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen /* we have all of them. update refcounts. */
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen /* Reference all used files. */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen view->cur_offset = view->cur->hdr.file_seq == min_file_seq ?
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen /* Map the files only after we've found them all. Otherwise if we map
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen one file and then another file just happens to get rotated, we could
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen include both files in the view but skip the last transactions from
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen the first file.
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen We're mapping the files in reverse order so that _log_file_map()
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen can verify that prev_file_offset matches how far it actually managed
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen to sync the file. */
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen for (i = array_count(&view->file_refs); i > 0; i--) {
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen start_offset = file->hdr.file_seq == min_file_seq ?
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen end_offset = file->hdr.file_seq == max_file_seq ?
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen ret = mail_transaction_log_file_map(file, start_offset,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen /* this file resets the index.
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen don't bother reading the others. */
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen view->prev_file_seq = view->cur->hdr.file_seq;
1cf72a848805fcf014b01c9d3665b6a157846a21Timo Sirainen view->max_file_offset = I_MIN(max_file_offset, view->head->sync_offset);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen if (mail_transaction_log_file_get_highest_modseq_at(view->cur,
fadd878cd6098f5b873c21c121209a922679dae4Timo Sirainen i_assert(view->cur_offset <= view->cur->sync_offset);
c680a6b35b459045e92814778908da5a93922107Timo Sirainenvoid mail_transaction_log_view_clear(struct mail_transaction_log_view *view,
f537e7efaec891d6b3320ca94331d09ca8c4a4dbTimo Sirainen if (mail_transaction_log_find_file(view->log, oldest_file_seq, FALSE,
ae8817f05005f57bba32479a610b52d083e2b6ebTimo Sirainen view->min_file_offset = view->max_file_offset = 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view,
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenmail_transaction_log_view_get_prev_modseq(struct mail_transaction_log_view *view)
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainenmail_transaction_log_view_get_last(struct mail_transaction_log_view *view,
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen struct mail_transaction_log_file *cur = view->cur;
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen if (cur->hdr.file_seq == view->max_file_seq) {
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen /* last file */
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen /* we're all finished */
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen /* end of file, go to next one */
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen /* not EOF */
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainenbool mail_transaction_log_view_is_last(struct mail_transaction_log_view *view)
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen return mail_transaction_log_view_get_last(view, &cur, &cur_offset);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen const char *fmt, ...)
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen mail_transaction_log_file_set_corrupted(view->log->head, "%s",
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenmail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view)
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainenlog_view_is_record_valid(struct mail_transaction_log_file *file,
b5ac20e30146562322ceb7939f044d52d1e51184Timo Sirainen rec_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK;
075912b4566a79c7bc59bf229c9f629ef7be0ea2Timo Sirainen rec_size = mail_index_offset_to_uint32(hdr->size) - sizeof(*hdr);
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen /* we want to be extra careful with expunges */
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen "expunge record missing protection mask");
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "Empty record contents (type=0x%x)", rec_type);
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen /* records that are exported by syncing and view syncing will be
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen checked here so that we don't have to implement the same validation
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen multiple times. other records are checked internally by
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen mail_index_sync_record(). */
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen if ((rec_size % sizeof(struct mail_index_record)) != 0) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen "Invalid append record size");
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen uid_buf = buffer_create_const_data(pool_datastack_create(),
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen uid_buf = buffer_create_const_data(pool_datastack_create(),
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen const struct mail_transaction_keyword_update *rec = data;
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen seqset_offset = sizeof(*rec) + rec->name_size;
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen "Invalid keyword update record size");
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen uid_buf = buffer_create_const_data(pool_datastack_create(),
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen array_create_from_buffer(&uids, uid_buf, sizeof(uint32_t)*2);
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen uid_buf = buffer_create_const_data(pool_datastack_create(),
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen sizeof(struct mail_transaction_keyword_reset));
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen if ((uid_buf->used % uids.arr.element_size) != 0) {
a5b331e18b220fac557480b569b85215a1b3bd8eTimo Sirainen } else if (count == 0) {
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen if (rec->seq1 > rec->seq2 || rec->seq1 == 0) {
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen "Invalid UID range "
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen "(%u .. %u, type=0x%x)",
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen if (prev != NULL && rec->seq1 <= prev->seq2) {
c91de2744f8c1e61e91082ff5e214450f28a0e7cTimo Sirainen "Non-sorted UID ranges (type=0x%x)",
a8012fea2a7315033bc467acbf46be8e7323318cTimo Sirainenlog_view_get_next(struct mail_transaction_log_view *view,
834b90e1f426d1e3308670e09c050bcdea546eb8Timo Sirainen const void **data_r)
fadd878cd6098f5b873c21c121209a922679dae4Timo Sirainen /* prev_file_offset should point to beginning of previous log record.
fadd878cd6098f5b873c21c121209a922679dae4Timo Sirainen when we reach EOF, it should be left there, not to beginning of the
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen next file that's not included inside the view. */
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen if (mail_transaction_log_view_get_last(view, &view->cur,
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen /* if the last file was the beginning of a file, we want to
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen move prev pointers there */
c9c24293550541307f1bb41bba4a0fdfe2fa59e0Timo Sirainen view->prev_file_seq = view->cur->hdr.file_seq;
f2786c07cbd4a7a0a6a46c3e06dc4545aaf2f278Timo Sirainen view->prev_file_seq = view->cur->hdr.file_seq;
f1e1d821d93e4a1dc6ed8f23febde868b5d64cd5Timo Sirainen data = buffer_get_data(file->buffer, &file_size);
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen if (view->cur_offset + sizeof(*hdr) > file_size) {
f1e1d821d93e4a1dc6ed8f23febde868b5d64cd5Timo Sirainen "offset points outside file "
f1e1d821d93e4a1dc6ed8f23febde868b5d64cd5Timo Sirainen "(%"PRIuUOFF_T" + %"PRIuSIZE_T" > %"PRIuSIZE_T")",
6eb30032b4a50c383dea4c9c74342d906de6ad36Timo Sirainen i_assert(view->cur_offset >= file->buffer_offset);
a24f6b02ed8d0dde933a715be1c86f01977bf610Timo Sirainen hdr = CONST_PTR_OFFSET(data, view->cur_offset - file->buffer_offset);
b5ac20e30146562322ceb7939f044d52d1e51184Timo Sirainen rec_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK;
075912b4566a79c7bc59bf229c9f629ef7be0ea2Timo Sirainen full_size = mail_index_offset_to_uint32(hdr->size);
1225a5a7ce39f1d5545d5ed3b84ecd4f72438d36Timo Sirainen "record size too small (type=0x%x, "
075912b4566a79c7bc59bf229c9f629ef7be0ea2Timo Sirainen if (file_size - view->cur_offset < full_size) {
1225a5a7ce39f1d5545d5ed3b84ecd4f72438d36Timo Sirainen "record size too large (type=0x%x, "
1225a5a7ce39f1d5545d5ed3b84ecd4f72438d36Timo Sirainen "offset=%"PRIuUOFF_T", size=%u, end=%"PRIuSIZE_T")",
b5ac20e30146562322ceb7939f044d52d1e51184Timo Sirainen rec_type, view->cur_offset, full_size, file_size);
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
85144b5f0bc763de14c7d87291a90ef74ac241a2Timo Sirainen if (mail_transaction_header_has_modseq(hdr, data,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenint mail_transaction_log_view_next(struct mail_transaction_log_view *view,
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen const void **data_r)
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen /* drop expunge protection */
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT))
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen view->tmp_hdr.type = hdr->type & ~MAIL_TRANSACTION_EXPUNGE_PROT;
8bae533c96e129dca8ee7b494d7de5aeb4a043a2Timo Sirainen /* return record's size */
5f78b33aa505b17e23cdf27b071a24e127b3db54Timo Sirainen view->tmp_hdr.size = mail_index_offset_to_uint32(hdr->size);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenvoid mail_transaction_log_view_mark(struct mail_transaction_log_view *view)
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainen i_assert(view->cur->hdr.file_seq == view->prev_file_seq);
95a1a5195d56f3cf5d1e529aad668f87ad3b979bTimo Sirainenvoid mail_transaction_log_view_rewind(struct mail_transaction_log_view *view)