bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* CRC32 of this entire record (except this checksum) */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Size of this entire record */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Mailbox GUID */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* { uid1, uid2 } pairs */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_uid_ranges[]; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* Total number of messages expunged so far in this log */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_count; */
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen HASH_TABLE(uint8_t *, struct fts_expunge_log_mailbox *) mailboxes;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstruct fts_expunge_log *fts_expunge_log_init(const char *path)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenvoid fts_expunge_log_deinit(struct fts_expunge_log **_log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenstatic int fts_expunge_log_open(struct fts_expunge_log *log, bool create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* FIXME: use proper permissions */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fd = open(log->path, O_RDWR | O_APPEND | (create ? O_CREAT : 0), 0600);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_reopen_if_needed(struct fts_expunge_log *log, bool create)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* same file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* file changed */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* recreate the file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_expunge_count(struct fts_expunge_log *log,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((uoff_t)log->st.st_size < sizeof(*expunge_count_r)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* we'll assume that write()s atomically grow the file size, as
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen O_APPEND almost guarantees. even if not, having a race condition
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen isn't the end of the world. the expunge count is simply read wrong
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen and fts optimize is performed earlier or later than intended. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ret = pread(log->fd, expunge_count_r, sizeof(*expunge_count_r),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("pread(%s) read only %d of %d bytes", log->path,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_append_begin(struct fts_expunge_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen pool = pool_alloconly_create("fts expunge log append", 1024);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx = p_new(pool, struct fts_expunge_log_append_ctx, 1);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_create(&ctx->mailboxes, pool, 0, guid_128_hash, guid_128_cmp);
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody if (log != NULL && fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_mailbox_alloc(struct fts_expunge_log_append_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = p_new(ctx->pool, struct fts_expunge_log_mailbox, 1);
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen hash_table_insert(ctx->mailboxes, guid_p, mailbox);
2282944443f78bd186809df235da1b7e801f0430Phil Carmodyfts_expunge_log_append_mailbox(struct fts_expunge_log_append_ctx *ctx,
6b2f82c7ed4e646217a1ee29e99c83d95762a8d6Phil Carmody guid_128_equals(mailbox_guid, ctx->prev_mailbox->guid))
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen mailbox = hash_table_lookup(ctx->mailboxes, guid_p);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen mailbox = fts_expunge_log_mailbox_alloc(ctx, mailbox_guid);
2282944443f78bd186809df235da1b7e801f0430Phil Carmodyvoid fts_expunge_log_append_next(struct fts_expunge_log_append_ctx *ctx,
2282944443f78bd186809df235da1b7e801f0430Phil Carmody mailbox = fts_expunge_log_append_mailbox(ctx, mailbox_guid);
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if (!seq_range_array_add(&mailbox->uids, uid))
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmodyvoid fts_expunge_log_append_range(struct fts_expunge_log_append_ctx *ctx,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody mailbox = fts_expunge_log_append_mailbox(ctx, mailbox_guid);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody mailbox->uids_count += seq_range_array_add_range_count(&mailbox->uids,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody /* To be honest, an unbacked log doesn't need to maintain the uids_count,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody but we don't know here if we're supporting an unbacked log or not, so we
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody have to maintain the value, just in case.
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody At the moment, the only caller of this function is for unbacked logs. */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmodyvoid fts_expunge_log_append_record(struct fts_expunge_log_append_ctx *ctx,
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const struct fts_expunge_log_read_record *record)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody /* FIXME: Optimise with a merge */
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody fts_expunge_log_append_range(ctx, record->mailbox_guid, range);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmodystatic void fts_expunge_log_append_mailbox_record(struct fts_expunge_log_append_ctx *ctx,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody /* FIXME: Optimise with a merge */
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody fts_expunge_log_append_range(ctx, mailbox->guid, range);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_export(struct fts_expunge_log_append_ctx *ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen iter = hash_table_iterate_init(ctx->mailboxes);
a75d470c9223a75801418fcdda258885c36317e0Timo Sirainen while (hash_table_iterate(iter, ctx->mailboxes, &guid_p, &mailbox)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = buffer_append_space_unsafe(output, sizeof(*rec));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen memcpy(rec->guid, mailbox->guid, sizeof(rec->guid));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_uid_ranges[]; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_append(output, array_idx(&mailbox->uids, 0),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(struct seq_range));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* uint32_t expunge_count; */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buffer_append(output, &expunge_count, sizeof(expunge_count));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* update the header now that we know the record contents */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen rec = buffer_get_space_unsafe(output, rec_offset,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_write(struct fts_expunge_log_append_ctx *ctx)
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmody /* Unbacked expunge logs cannot be written, by definition */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* try to append to the latest file */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_read_expunge_count(log, &expunge_count) < 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen buf = buffer_create_dynamic(default_pool, 1024);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_export(ctx, expunge_count, buf);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* the file was opened with O_APPEND, so this write() should be
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen appended atomically without any need for locking. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((ret = write_full(log->fd, buf->data, buf->used)) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen i_error("ftruncate(%s) failed: %m", log->path);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if ((ret = fts_expunge_log_reopen_if_needed(log, TRUE)) <= 0)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* the log was unlinked, so we'll need to write again to
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen the new file. the expunge_count needs to be reset to zero
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen from here. */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen e = buffer_get_space_unsafe(buf, buf->used - sizeof(uint32_t),
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* finish by closing the log. this forces NFS to flush the
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen changes to disk without our having to explicitly play with
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* FIXME: we should ftruncate() in case there
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen were partial writes.. */
21bd3573f06cec26de53487cbbdf84d1e328a47dAki Tuomistatic int fts_expunge_log_append_finalize(struct fts_expunge_log_append_ctx **_ctx,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen struct fts_expunge_log_append_ctx *ctx = *_ctx;
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainenint fts_expunge_log_uid_count(struct fts_expunge_log *log,
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen unsigned int *expunges_r)
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen if ((ret = fts_expunge_log_reopen_if_needed(log, FALSE)) <= 0) {
1d3ee23f5f573d5e5ab79422c0a23dab85f3d034Timo Sirainen return fts_expunge_log_read_expunge_count(log, expunges_r);
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodyint fts_expunge_log_append_commit(struct fts_expunge_log_append_ctx **_ctx)
21bd3573f06cec26de53487cbbdf84d1e328a47dAki Tuomi return fts_expunge_log_append_finalize(_ctx, TRUE);
2442c230b6c95b20d26ead752f9de1cb40544a53Phil Carmodyint fts_expunge_log_append_abort(struct fts_expunge_log_append_ctx **_ctx)
21bd3573f06cec26de53487cbbdf84d1e328a47dAki Tuomi return fts_expunge_log_append_finalize(_ctx, FALSE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_begin(struct fts_expunge_log *log)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen ctx = i_new(struct fts_expunge_log_read_ctx, 1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (fts_expunge_log_reopen_if_needed(log, FALSE) < 0)
e93184a9055c2530366dfe617e07199603c399ddMartti Rannanjärvi ctx->input = i_stream_create_fd(log->fd, (size_t)-1);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_record_size_is_valid(const struct fts_expunge_log_record *rec,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen unsigned int *uids_size_r)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (rec->record_size < sizeof(*rec) + sizeof(uint32_t)*3)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen *uids_size_r = rec->record_size - sizeof(*rec) - sizeof(uint32_t);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen return *uids_size_r % sizeof(uint32_t)*2 == 0;
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_failure(struct fts_expunge_log_read_ctx *ctx,
0f3d4fbcf88e2ffd674893aed8cc1288fe17d290Timo Sirainen i_error("read(%s) failed: %s", ctx->log->path,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Unexpected EOF (read %"PRIuSIZE_T" / %u bytes)",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenfts_expunge_log_read_next(struct fts_expunge_log_read_ctx *ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* initial read to try to get the record */
d868a04630bd7bfe9c1543a7c3f68703b3e276e4Timo Sirainen (void)i_stream_read_bytes(ctx->input, &data, &size, IO_BLOCK_SIZE);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size == 0 && ctx->input->stream_errno == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* expected EOF - mark the file as read by unlinking it */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* try reading again, in case something new was written */
d868a04630bd7bfe9c1543a7c3f68703b3e276e4Timo Sirainen (void)i_stream_read_bytes(ctx->input, &data, &size,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (size == 0 && ctx->input->stream_errno == 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* expected EOF */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_read_failure(ctx, sizeof(*rec));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen if (!fts_expunge_log_record_size_is_valid(rec, &uids_size)) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Invalid record size: %u",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* read the entire record */
573424407a2d3c1453638a643583a7cf10c129e1Phil Carmody if (i_stream_read_bytes(ctx->input, &data, &size, rec->record_size) < 0) {
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen fts_expunge_log_read_failure(ctx, rec->record_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* verify that the record checksum is valid */
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen "Record checksum mismatch: %u != %u",
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen /* create the UIDs array by pointing it directly into input
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen stream's buffer */
3281669db44d09a087a203201248abbc81b3cc1aTimo Sirainen buffer_create_from_const_data(&ctx->buffer, rec + 1, uids_size);
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen array_create_from_buffer(&ctx->read_rec.uids, &ctx->buffer,
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen sizeof(struct seq_range));
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainenint fts_expunge_log_read_end(struct fts_expunge_log_read_ctx **_ctx)
cc2954ad6d8ba0509b870d773ba4b6b16353763cTimo Sirainen int ret = ctx->failed ? -1 : (ctx->corrupted ? 0 : 1);
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody struct fts_expunge_log_append_ctx **flattened_r)
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody const struct fts_expunge_log_read_record *record;
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody while((record = fts_expunge_log_read_next(read_ctx)) != NULL) {
8d4428cee277a9a84450a8c6771f7a4dc68638cfPhil Carmody if ((ret = fts_expunge_log_read_end(&read_ctx)) > 0)
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmodybool fts_expunge_log_contains(const struct fts_expunge_log_append_ctx *ctx,
5433d56d338db3948bf7bc22ef1daaba147dbb5dPhil Carmody mailbox = hash_table_lookup(ctx->mailboxes, guid_p);
ea125b1c394cb6a281c6120f5396faa11fcb268bPhil Carmodyint fts_expunge_log_append_remove(struct fts_expunge_log_append_ctx *from,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody const struct fts_expunge_log_read_record *record)
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody struct fts_expunge_log_mailbox *mailbox = hash_table_lookup(from->mailboxes, guid_p);
ea125b1c394cb6a281c6120f5396faa11fcb268bPhil Carmody return 0; /* may only remove things that exist */
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody mailbox->uids_count -= seq_range_array_remove_seq_range(&mailbox->uids, &record->uids);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmodyint fts_expunge_log_subtract(struct fts_expunge_log_append_ctx *from,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody struct fts_expunge_log_read_ctx *read_ctx = fts_expunge_log_read_begin(subtract);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody const struct fts_expunge_log_read_record *record;
ea125b1c394cb6a281c6120f5396faa11fcb268bPhil Carmody while ((record = fts_expunge_log_read_next(read_ctx)) != NULL) {
ea125b1c394cb6a281c6120f5396faa11fcb268bPhil Carmody if (fts_expunge_log_append_remove(from, record) <= 0)
ea125b1c394cb6a281c6120f5396faa11fcb268bPhil Carmody i_warning("fts: Expunge log subtract ignored %u nonexistent mailbox GUIDs",
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody/* It could be argued that somehow adding a log (file) to the append context
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody and then calling the _write() helper would be easier. But then there's the
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody _commit() vs. _abort() cleanup that would need to be addressed. Just creating
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody a copy is simpler. */
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmodyint fts_expunge_log_flat_write(const struct fts_expunge_log_append_ctx *read_log,
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody const char *path)
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody struct fts_expunge_log *nlog = fts_expunge_log_init(path);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody struct fts_expunge_log_append_ctx *nappend = fts_expunge_log_append_begin(nlog);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody iter = hash_table_iterate_init(read_log->mailboxes);
b22dcc3c0c84919fe500cf877d8dfd9bddbef76fPhil Carmody while (hash_table_iterate(iter, read_log->mailboxes, &guid_p, &mailbox))