fts-expunge-log.c revision 4307c886579381dbb1897ea1388ae6978c96f560
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2011-2012 Dovecot authors, see the included COPYING file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* CRC32 of this entire record (except this checksum) */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Size of this entire record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Mailbox GUID */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* { uid1, uid2 } pairs */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_uid_ranges[]; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Total number of messages expunged so far in this log */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_count; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log *fts_expunge_log_init(const char *path)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenvoid fts_expunge_log_deinit(struct fts_expunge_log **_log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int fts_expunge_log_open(struct fts_expunge_log *log, bool create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* FIXME: use proper permissions */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fd = open(log->path, O_RDWR | O_APPEND | (create ? O_CREAT : 0), 0600);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_reopen_if_needed(struct fts_expunge_log *log, bool create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* same file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* file changed */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* recreate the file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_expunge_count(struct fts_expunge_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((uoff_t)log->st.st_size < sizeof(*expunge_count_r)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* we'll assume that write()s atomically grow the file size, as
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen O_APPEND almost guarantees. even if not, having a race condition
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen isn't the end of the world. the expunge count is simply read wrong
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen and fts optimize is performed earlier or later than intended. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ret = pread(log->fd, expunge_count_r, sizeof(*expunge_count_r),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("pread(%s) read only %d of %d bytes", log->path,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_append_begin(struct fts_expunge_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen pool = pool_alloconly_create("fts expunge log append", 1024);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx = p_new(pool, struct fts_expunge_log_append_ctx, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_mailbox_alloc(struct fts_expunge_log_append_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = p_new(ctx->pool, struct fts_expunge_log_mailbox, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen memcpy(mailbox->guid, mailbox_guid, sizeof(mailbox->guid));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen hash_table_insert(ctx->mailboxes, mailbox->guid, mailbox);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenvoid fts_expunge_log_append_next(struct fts_expunge_log_append_ctx *ctx,
2ee478390151150a62dfd4a9d4e7b3a3d3a6da06Timo Sirainen memcmp(mailbox_guid, ctx->prev_mailbox->guid, GUID_128_SIZE) == 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = hash_table_lookup(ctx->mailboxes, mailbox_guid);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = fts_expunge_log_mailbox_alloc(ctx, mailbox_guid);
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if (!seq_range_array_add(&mailbox->uids, uid))
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_export(struct fts_expunge_log_append_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen iter = hash_table_iterate_init(ctx->mailboxes);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen while (hash_table_iterate(iter, &key, &value)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_mailbox *mailbox = value;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = buffer_append_space_unsafe(output, sizeof(*rec));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen memcpy(rec->guid, mailbox->guid, sizeof(rec->guid));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_uid_ranges[]; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_append(output, array_idx(&mailbox->uids, 0),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(struct seq_range));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_count; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_append(output, &expunge_count, sizeof(expunge_count));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* update the header now that we know the record contents */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = buffer_get_space_unsafe(output, rec_offset,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_write(struct fts_expunge_log_append_ctx *ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* try to append to the latest file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_read_expunge_count(log, &expunge_count) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buf = buffer_create_dynamic(default_pool, 1024);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_export(ctx, expunge_count, buf);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* the file was opened with O_APPEND, so this write() should be
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen appended atomically without any need for locking. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((ret = write_full(log->fd, buf->data, buf->used)) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("ftruncate(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((ret = fts_expunge_log_reopen_if_needed(log, TRUE)) <= 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* the log was unlinked, so we'll need to write again to
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen the new file. the expunge_count needs to be reset to zero
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen from here. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen e = buffer_get_space_unsafe(buf, buf->used - sizeof(uint32_t),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* finish by closing the log. this forces NFS to flush the
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen changes to disk without our having to explicitly play with
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* FIXME: we should ftruncate() in case there
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen were partial writes.. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenint fts_expunge_log_append_commit(struct fts_expunge_log_append_ctx **_ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_append_ctx *ctx = *_ctx;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenint fts_expunge_log_uid_count(struct fts_expunge_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned int *expunges_r)
bc0553df7b8d654eb2e8ba434692f487e05abd02Timo Sirainen if ((ret = fts_expunge_log_reopen_if_needed(log, FALSE)) <= 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return fts_expunge_log_read_expunge_count(log, expunges_r);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_begin(struct fts_expunge_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx = i_new(struct fts_expunge_log_read_ctx, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, FALSE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx->input = i_stream_create_fd(log->fd, (size_t)-1, FALSE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_record_size_is_valid(const struct fts_expunge_log_record *rec,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned int *uids_size_r)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (rec->record_size < sizeof(*rec) + sizeof(uint32_t)*3)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *uids_size_r = rec->record_size - sizeof(*rec) - sizeof(uint32_t);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return *uids_size_r % sizeof(uint32_t)*2 == 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_failure(struct fts_expunge_log_read_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("read(%s) failed: %m", ctx->log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Unexpected EOF (read %"PRIuSIZE_T" / %u bytes)",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_next(struct fts_expunge_log_read_ctx *ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen const unsigned char *data;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* initial read to try to get the record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen (void)i_stream_read_data(ctx->input, &data, &size, IO_BLOCK_SIZE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size == 0 && ctx->input->stream_errno == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* expected EOF - mark the file as read by unlinking it */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (unlink(ctx->log->path) < 0 && errno != ENOENT)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("unlink(%s) failed: %m", ctx->log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* try reading again, in case something new was written */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen (void)i_stream_read_data(ctx->input, &data, &size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size == 0 && ctx->input->stream_errno == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* expected EOF */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_read_failure(ctx, sizeof(*rec));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (!fts_expunge_log_record_size_is_valid(rec, &uids_size)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Invalid record size: %u",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* read the entire record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (i_stream_read_data(ctx->input, &data, &size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_read_failure(ctx, rec->record_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* verify that the record checksum is valid */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Record checksum mismatch: %u != %u",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* create the UIDs array by pointing it directly into input
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen stream's buffer */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_create_const_data(&ctx->buffer, rec + 1, uids_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen array_create_from_buffer(&ctx->read_rec.uids, &ctx->buffer,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(struct seq_range));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenint fts_expunge_log_read_end(struct fts_expunge_log_read_ctx **_ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int ret = ctx->failed ? -1 : (ctx->corrupted ? 0 : 1);