fts-expunge-log.c revision 8d4428cee277a9a84450a8c6771f7a4dc68638cf
7cb128dc4cae2a03a742f63ba7afee23c78e3af0Phil Carmody/* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "lib.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "array.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "crc32.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "hash.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "istream.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "write-full.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "seq-range-array.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "mail-storage.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include "fts-expunge-log.h"
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include <sys/stat.h>
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include <unistd.h>
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen#include <fcntl.h>
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log_record {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* CRC32 of this entire record (except this checksum) */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen uint32_t checksum;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Size of this entire record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen uint32_t record_size;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Mailbox GUID */
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen guid_128_t guid;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* { uid1, uid2 } pairs */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_uid_ranges[]; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Total number of messages expunged so far in this log */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_count; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen};
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen char *path;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int fd;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct stat st;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen};
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log_mailbox {
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen guid_128_t guid;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ARRAY_TYPE(seq_range) uids;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned uids_count;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen};
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log_append_ctx {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log *log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen pool_t pool;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE(uint8_t *, struct fts_expunge_log_mailbox *) mailboxes;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_mailbox *prev_mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen bool failed;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen};
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log_read_ctx {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log *log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct istream *input;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_t buffer;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_read_record read_rec;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen bool failed;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen bool corrupted;
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody bool unlink;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen};
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log *fts_expunge_log_init(const char *path)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log *log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log = i_new(struct fts_expunge_log, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log->path = i_strdup(path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log->fd = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenvoid fts_expunge_log_deinit(struct fts_expunge_log **_log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log *log = *_log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *_log = NULL;
d06e111700769927a64b46dc157ac1c122dff1a0Timo Sirainen if (log->fd != -1)
d06e111700769927a64b46dc157ac1c122dff1a0Timo Sirainen i_close_fd(&log->fd);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_free(log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_free(log);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int fts_expunge_log_open(struct fts_expunge_log *log, bool create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int fd;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_assert(log->fd == -1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* FIXME: use proper permissions */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fd = open(log->path, O_RDWR | O_APPEND | (create ? O_CREAT : 0), 0600);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fd == -1) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (errno == ENOENT && !create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("open(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fstat(fd, &log->st) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("fstat(%s) failed: %m", log->path);
4307c886579381dbb1897ea1388ae6978c96f560Timo Sirainen i_close_fd(&fd);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log->fd = fd;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return 1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_reopen_if_needed(struct fts_expunge_log *log, bool create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct stat st;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (log->fd == -1)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return fts_expunge_log_open(log, create);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (stat(log->path, &st) == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (st.st_ino == log->st.st_ino &&
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen CMP_DEV_T(st.st_dev, log->st.st_dev)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* same file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* file changed */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen } else if (errno == ENOENT) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* recreate the file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen } else {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("stat(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (close(log->fd) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("close(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log->fd = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return fts_expunge_log_open(log, create);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_expunge_count(struct fts_expunge_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen uint32_t *expunge_count_r)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ssize_t ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_assert(log->fd != -1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fstat(log->fd, &log->st) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("fstat(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((uoff_t)log->st.st_size < sizeof(*expunge_count_r)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *expunge_count_r = 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* we'll assume that write()s atomically grow the file size, as
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen O_APPEND almost guarantees. even if not, having a race condition
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen isn't the end of the world. the expunge count is simply read wrong
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen and fts optimize is performed earlier or later than intended. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ret = pread(log->fd, expunge_count_r, sizeof(*expunge_count_r),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log->st.st_size - 4);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ret < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("pread(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ret != sizeof(*expunge_count_r)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("pread(%s) read only %d of %d bytes", log->path,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen (int)ret, (int)sizeof(*expunge_count_r));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log_append_ctx *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_append_begin(struct fts_expunge_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_append_ctx *ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen pool_t pool;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen pool = pool_alloconly_create("fts expunge log append", 1024);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx = p_new(pool, struct fts_expunge_log_append_ctx, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->log = log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->pool = pool;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&ctx->mailboxes, pool, 0, guid_128_hash, guid_128_cmp);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody if (log != NULL && fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->failed = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic struct fts_expunge_log_mailbox *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_mailbox_alloc(struct fts_expunge_log_append_ctx *ctx,
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen const guid_128_t mailbox_guid)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen uint8_t *guid_p;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_mailbox *mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = p_new(ctx->pool, struct fts_expunge_log_mailbox, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen memcpy(mailbox->guid, mailbox_guid, sizeof(mailbox->guid));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen p_array_init(&mailbox->uids, ctx->pool, 16);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen guid_p = mailbox->guid;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_insert(ctx->mailboxes, guid_p, mailbox);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
2282944443f78bd186809df235da1b7e801f0430Phil Carmodystatic struct fts_expunge_log_mailbox *
2282944443f78bd186809df235da1b7e801f0430Phil Carmodyfts_expunge_log_append_mailbox(struct fts_expunge_log_append_ctx *ctx,
2282944443f78bd186809df235da1b7e801f0430Phil Carmody const guid_128_t mailbox_guid)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen const uint8_t *guid_p = mailbox_guid;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_mailbox *mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ctx->prev_mailbox != NULL &&
2ee478390151150a62dfd4a9d4e7b3a3d3a6da06Timo Sirainen memcmp(mailbox_guid, ctx->prev_mailbox->guid, GUID_128_SIZE) == 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = ctx->prev_mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen else {
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen mailbox = hash_table_lookup(ctx->mailboxes, guid_p);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (mailbox == NULL)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = fts_expunge_log_mailbox_alloc(ctx, mailbox_guid);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->prev_mailbox = mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
2282944443f78bd186809df235da1b7e801f0430Phil Carmody return mailbox;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody}
2282944443f78bd186809df235da1b7e801f0430Phil Carmodyvoid fts_expunge_log_append_next(struct fts_expunge_log_append_ctx *ctx,
2282944443f78bd186809df235da1b7e801f0430Phil Carmody const guid_128_t mailbox_guid,
2282944443f78bd186809df235da1b7e801f0430Phil Carmody uint32_t uid)
2282944443f78bd186809df235da1b7e801f0430Phil Carmody{
2282944443f78bd186809df235da1b7e801f0430Phil Carmody struct fts_expunge_log_mailbox *mailbox;
2282944443f78bd186809df235da1b7e801f0430Phil Carmody
2282944443f78bd186809df235da1b7e801f0430Phil Carmody mailbox = fts_expunge_log_append_mailbox(ctx, mailbox_guid);
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if (!seq_range_array_add(&mailbox->uids, uid))
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox->uids_count++;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmodyvoid fts_expunge_log_append_range(struct fts_expunge_log_append_ctx *ctx,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const guid_128_t mailbox_guid,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const struct seq_range *uids)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody{
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody struct fts_expunge_log_mailbox *mailbox;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody mailbox = fts_expunge_log_append_mailbox(ctx, mailbox_guid);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody mailbox->uids_count += seq_range_array_add_range_count(&mailbox->uids,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody uids->seq1, uids->seq2);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody /* To be honest, an unbacked log doesn't need to maintain the uids_count,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody but we don't know here if we're supporting an unbacked log or not, so we
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody have to maintain the value, just in case.
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody At the moment, the only caller of this function is for unbacked logs. */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody}
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmodyvoid fts_expunge_log_append_record(struct fts_expunge_log_append_ctx *ctx,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const struct fts_expunge_log_read_record *record)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody{
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const struct seq_range *range;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody /* FIXME: Optimise with a merge */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody array_foreach(&record->uids, range)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody fts_expunge_log_append_range(ctx, record->mailbox_guid, range);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic void
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_export(struct fts_expunge_log_append_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen uint32_t expunge_count, buffer_t *output)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct hash_iterate_context *iter;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen uint8_t *guid_p;
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen struct fts_expunge_log_mailbox *mailbox;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_record *rec;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen size_t rec_offset;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen iter = hash_table_iterate_init(ctx->mailboxes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, ctx->mailboxes, &guid_p, &mailbox)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec_offset = output->used;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = buffer_append_space_unsafe(output, sizeof(*rec));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen memcpy(rec->guid, mailbox->guid, sizeof(rec->guid));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_uid_ranges[]; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_append(output, array_idx(&mailbox->uids, 0),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen array_count(&mailbox->uids) *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(struct seq_range));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_count; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen expunge_count += mailbox->uids_count;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_append(output, &expunge_count, sizeof(expunge_count));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* update the header now that we know the record contents */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = buffer_get_space_unsafe(output, rec_offset,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen output->used - rec_offset);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec->record_size = output->used - rec_offset;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec->checksum = crc32_data(&rec->record_size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec->record_size -
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(rec->checksum));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen hash_table_iterate_deinit(&iter);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_write(struct fts_expunge_log_append_ctx *ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log *log = ctx->log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_t *buf;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen uint32_t expunge_count, *e;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody /* Unbacked expunge logs cannot be written, by definition */
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody i_assert(log != NULL);
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* try to append to the latest file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_read_expunge_count(log, &expunge_count) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buf = buffer_create_dynamic(default_pool, 1024);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_export(ctx, expunge_count, buf);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* the file was opened with O_APPEND, so this write() should be
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen appended atomically without any need for locking. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen for (;;) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((ret = write_full(log->fd, buf->data, buf->used)) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("write(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ftruncate(log->fd, log->st.st_size) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("ftruncate(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((ret = fts_expunge_log_reopen_if_needed(log, TRUE)) <= 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen break;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* the log was unlinked, so we'll need to write again to
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen the new file. the expunge_count needs to be reset to zero
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen from here. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen e = buffer_get_space_unsafe(buf, buf->used - sizeof(uint32_t),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(uint32_t));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_assert(*e > expunge_count);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *e -= expunge_count;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen expunge_count = 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_free(&buf);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ret == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* finish by closing the log. this forces NFS to flush the
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen changes to disk without our having to explicitly play with
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fsync() */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (close(log->fd) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* FIXME: we should ftruncate() in case there
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen were partial writes.. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("close(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ret = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen log->fd = -1;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodystatic int fts_expunge_log_append_finalise(struct fts_expunge_log_append_ctx **_ctx,
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody bool commit)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_append_ctx *ctx = *_ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int ret = ctx->failed ? -1 : 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *_ctx = NULL;
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody if (commit && ret == 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ret = fts_expunge_log_write(ctx);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen hash_table_destroy(&ctx->mailboxes);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen pool_unref(&ctx->pool);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodyint fts_expunge_log_append_commit(struct fts_expunge_log_append_ctx **_ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody return fts_expunge_log_append_finalise(_ctx, TRUE);
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody}
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodyint fts_expunge_log_append_abort(struct fts_expunge_log_append_ctx **_ctx)
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody{
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody return fts_expunge_log_append_finalise(_ctx, FALSE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log_read_ctx *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_begin(struct fts_expunge_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_read_ctx *ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx = i_new(struct fts_expunge_log_read_ctx, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->log = log;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, FALSE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->failed = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen else if (log->fd != -1)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->input = i_stream_create_fd(log->fd, (size_t)-1, FALSE);
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody ctx->unlink = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic bool
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_record_size_is_valid(const struct fts_expunge_log_record *rec,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned int *uids_size_r)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (rec->record_size < sizeof(*rec) + sizeof(uint32_t)*3)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return FALSE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *uids_size_r = rec->record_size - sizeof(*rec) - sizeof(uint32_t);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return *uids_size_r % sizeof(uint32_t)*2 == 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic void
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_failure(struct fts_expunge_log_read_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned int wanted_size)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen size_t size;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ctx->input->stream_errno != 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->failed = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("read(%s) failed: %m", ctx->log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen } else {
3785910c303507db5f629684e6dde2cc7f83668eTimo Sirainen size = i_stream_get_data_size(ctx->input);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->corrupted = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("Corrupted fts expunge log %s: "
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Unexpected EOF (read %"PRIuSIZE_T" / %u bytes)",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->log->path, size, wanted_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenconst struct fts_expunge_log_read_record *
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_next(struct fts_expunge_log_read_ctx *ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen const unsigned char *data;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen const struct fts_expunge_log_record *rec;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned int uids_size;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen size_t size;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen uint32_t checksum;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ctx->input == NULL)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* initial read to try to get the record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen (void)i_stream_read_data(ctx->input, &data, &size, IO_BLOCK_SIZE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size == 0 && ctx->input->stream_errno == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* expected EOF - mark the file as read by unlinking it */
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody if (ctx->unlink &&
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody unlink(ctx->log->path) < 0 && errno != ENOENT)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("unlink(%s) failed: %m", ctx->log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* try reading again, in case something new was written */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_stream_sync(ctx->input);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen (void)i_stream_read_data(ctx->input, &data, &size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen IO_BLOCK_SIZE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size < sizeof(*rec)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size == 0 && ctx->input->stream_errno == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* expected EOF */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_read_failure(ctx, sizeof(*rec));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = (const void *)data;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (!fts_expunge_log_record_size_is_valid(rec, &uids_size)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->corrupted = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("Corrupted fts expunge log %s: "
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Invalid record size: %u",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->log->path, rec->record_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* read the entire record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen while (size < rec->record_size) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (i_stream_read_data(ctx->input, &data, &size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec->record_size-1) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_read_failure(ctx, rec->record_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = (const void *)data;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* verify that the record checksum is valid */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen checksum = crc32_data(&rec->record_size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec->record_size - sizeof(rec->checksum));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (checksum != rec->checksum) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->corrupted = TRUE;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("Corrupted fts expunge log %s: "
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Record checksum mismatch: %u != %u",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->log->path, checksum, rec->checksum);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen }
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen memcpy(ctx->read_rec.mailbox_guid, rec->guid,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(ctx->read_rec.mailbox_guid));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* create the UIDs array by pointing it directly into input
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen stream's buffer */
3281669db44d09a087a203201248abbc81b3cc1aTimo Sirainen buffer_create_from_const_data(&ctx->buffer, rec + 1, uids_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen array_create_from_buffer(&ctx->read_rec.uids, &ctx->buffer,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(struct seq_range));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_stream_skip(ctx->input, rec->record_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return &ctx->read_rec;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenint fts_expunge_log_read_end(struct fts_expunge_log_read_ctx **_ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen{
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_read_ctx *ctx = *_ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int ret = ctx->failed ? -1 : (ctx->corrupted ? 0 : 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *_ctx = NULL;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen if (ctx->corrupted) {
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody if (ctx->unlink &&
87b5c1fc03945708726c175d540fd307f7f78480Phil Carmody unlink(ctx->log->path) < 0 && errno != ENOENT)
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen i_error("unlink(%s) failed: %m", ctx->log->path);
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen }
9afa8aff5bda7c24cce6cc19b8232d2a228b4109Timo Sirainen
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (ctx->input != NULL)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_stream_unref(&ctx->input);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_free(ctx);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return ret;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen}
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmodyint fts_expunge_log_flatten(const char *path,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody struct fts_expunge_log_append_ctx **flattened_r)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody{
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody struct fts_expunge_log *read;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody struct fts_expunge_log_read_ctx *read_ctx;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const struct fts_expunge_log_read_record *record;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody struct fts_expunge_log_append_ctx *append;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody int ret;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody i_assert(path != NULL && flattened_r != NULL);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody read = fts_expunge_log_init(path);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody read_ctx = fts_expunge_log_read_begin(read);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody read_ctx->unlink = FALSE;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody append = fts_expunge_log_append_begin(NULL);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody while((record = fts_expunge_log_read_next(read_ctx)) != NULL) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody fts_expunge_log_append_record(append, record);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody }
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody if ((ret = fts_expunge_log_read_end(&read_ctx)) > 0)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody *flattened_r = append;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody fts_expunge_log_deinit(&read);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody return ret;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody}