mail-index-strmap.c revision 31a574fda352ef4f71dbff9c30e15e4744e132c0
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2008-2012 Dovecot authors, see the included COPYING file */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data, *end, *str_idx_base;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const char *str;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen/* number of bytes required to store one string idx */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#define STRMAP_FILE_STRIDX_SIZE (sizeof(uint32_t)*2)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen/* renumber the string indexes when highest string idx becomes larger than
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen <number of indexes>*STRMAP_FILE_MAX_STRIDX_MULTIPLIER */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#define STRIDX_MUST_RENUMBER(highest_idx, n_unique_indexes) \
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (highest_idx > n_unique_indexes * STRMAP_FILE_MAX_STRIDX_MULTIPLIER)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenconst struct dotlock_settings default_dotlock_settings = {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_init(struct mail_index *index, const char *suffix)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->path = i_strconcat(index->filepath, suffix, NULL);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->dotlock_settings = default_dotlock_settings;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_set_syscall_error(struct mail_index_strmap *strmap,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "%s failed with strmap index file %s: %m",
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_close(struct mail_index_strmap *strmap)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "close()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_deinit(struct mail_index_strmap **_strmap)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic unsigned int mail_index_strmap_hash_key(const void *_key)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_hash_key *key = _key;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_hash_cmp(const void *_key, const void *_value, void *context)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_hash_key *key = _key;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_rec *rec = _value;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = context;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return view->key_compare(key->str, rec, view->cb_context);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_view_open(struct mail_index_strmap *strmap,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const ARRAY_TYPE(mail_index_strmap_rec) **recs_r,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view = i_new(struct mail_index_strmap_view, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->hash = hash2_create(0, sizeof(struct mail_index_strmap_rec),
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_close(struct mail_index_strmap_view **_view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenuint32_t mail_index_strmap_view_get_highest_idx(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_view_reset(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_view_set_corrupted(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Corrupted strmap index file: %s",
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_open(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "open()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->input = i_stream_create_fd(strmap->fd, (size_t)-1, FALSE);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = i_stream_read_data(strmap->input, &data, &size, sizeof(hdr)-1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "read()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (hdr.version != MAIL_INDEX_STRMAP_VERSION ||
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* need to rebuild. if we already had something in the strmap,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen we can keep it. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we'll read the entire file from the beginning */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_block_offset = sizeof(struct mail_index_strmap_header);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic bool mail_index_strmap_need_reopen(struct mail_index_strmap *strmap)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: nfs flush */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "fstat()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "stat()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return st1.st_ino != st2.st_ino || !CMP_DEV_T(st1.st_dev, st2.st_dev);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_refresh(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_strmap_need_reopen(view->strmap)) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* last read failed because view had a message
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen that didn't exist in the strmap (because it
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen was expunged by another session). if the
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen message still isn't expunged in this view,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen just continue using the current strmap. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* our view isn't synced with the disk, we
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen can't read strmap without first resetting
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_packed(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = i_stream_read_data(ctx->input, &data, &size, sizeof(*num_r) - 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->input->v_offset + size > ctx->end_offset)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size = ctx->end_offset - ctx->input->v_offset;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_unpack_num(&p, end, num_r) < 0)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_uid_exists(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->uid_lookup_idx >= ctx->view->view->map->hdr.messages_count) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (uid >= ctx->view->view->map->hdr.next_uid) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* thread index has larger UIDs than what we've seen
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen in our view. we'll have to read them again later
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen when we know about them */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec = MAIL_INDEX_MAP_IDX(ctx->view->view->map, ctx->uid_lookup_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* record that exists in index is missing from strmap.
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen see if it's because the strmap is corrupted or because
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen our current view is a bit stale and the message has already
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen been expunged. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (void)mail_index_refresh(ctx->view->view->index);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_rec_first(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* <uid> <n> <crc32>*count <str_idx>*count
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen n = 0 -> count=1 (only Message-ID:)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen n = 1 -> count=2 (Message-ID: + In-Reply-To:)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen n = 2+ -> count=n (Message-ID: + References:)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_read_packed(ctx, &n) <= 0)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->rec_size = count * (sizeof(ctx->rec.str_idx) + sizeof(*crc32_r));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_uid_exists(ctx, ctx->rec.uid);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (i_stream_read_data(ctx->view->strmap->input, &ctx->data, &size,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->str_idx_base = ctx->data + count * sizeof(uint32_t);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* this message has already been expunged, ignore it.
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen update highest string indexes anyway. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&str_idx, ctx->str_idx_base, sizeof(str_idx));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(ctx->view->strmap->input, ctx->rec_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* everything exists. save it. FIXME: these ref_index values
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen are thread index specific, perhaps something more generic
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen should be used some day */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->end = ctx->data + count * sizeof(*crc32_r);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_strmap_read_rec_next(ctx, crc32_r))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(ctx->view->strmap->input, ctx->rec_size);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: str_idx could be stored as packed relative values
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (first relative to highest_idx, the rest relative to the
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen previous str_idx) */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* read the record contents */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&ctx->rec.str_idx, ctx->str_idx_base, sizeof(ctx->rec.str_idx));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* get to the next record */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->str_idx_base += sizeof(ctx->rec.str_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_read_block_init(struct mail_index_strmap_view *view,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->last_read_uid + 1 >= view->view->map->hdr.next_uid) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* come back later when we know about the new UIDs */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = i_stream_read_data(strmap->input, &data, &size,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* no new data */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "read()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&block_size, data, sizeof(block_size));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = mail_index_offset_to_uint32(block_size) >> 2;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* the rest of the file is either not written, or the previous
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen write didn't finish */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(strmap->input, sizeof(block_size));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->end_offset = strmap->input->v_offset + block_size;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->end_offset < strmap->input->v_offset) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* block size too large */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: when reading multiple blocks we shouldn't have to calculate
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen this every time */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_lookup_seq_range(view->view, ctx->rec.uid, (uint32_t)-1,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen seq1 = mail_index_view_get_messages_count(view->view) + 1;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_read_block_next(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_read_rec_next(ctx, crc32_r))
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* get next UID */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->input->v_offset == ctx->end_offset) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* this block is done */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_read_packed(ctx, &uid_diff) < 0)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_read_rec_first(ctx, crc32_r);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } while (ret == 0);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_read_block_deinit(struct mail_index_strmap_read_context *ctx, int ret,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = ctx->view;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->highest_str_idx > view->total_ref_count) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* if all string indexes are unique, highest_str_index equals
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen total_ref_count. otherwise it's always lower. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Corrupted strmap index file %s: "
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "String indexes too high "
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "(highest=%u max=%u)",
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* our view contained a message that had since been expunged. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->lost_expunged_uid = ctx->lost_expunged_uid;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (ret < 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "read()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (update_block_offset && !ctx->too_large_uids) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_block_offset = strmap->input->v_offset;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->next_str_idx <= ctx->highest_str_idx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->next_str_idx = ctx->highest_str_idx + 1;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_view_sync_handle_conflict(struct mail_index_strmap_read_context *ctx,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* hopefully it's a message that has since been expunged */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_lookup_seq(ctx->view->view, hash_rec->uid, &seq)) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* message is no longer in our view. remove it completely. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_is_expunged(ctx->view->view, seq)) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* it's quite likely a conflict. we may not be able to verify
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen it, so just assume it is. nothing breaks even if we guess
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen wrong, the performance just suffers a bit. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* 0 means "doesn't match", which is the only acceptable case */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ctx->view->rec_compare(&ctx->rec, hash_rec,
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainenstrmap_view_sync_block_check_conflicts(struct mail_index_strmap_read_context *ctx,
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen /* unique string - there are no conflicts */
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen /* check for conflicting string indexes. they may happen if
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen 1) msgid exists only for a message X that has been expunged
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen 2) another process doesn't see X, but sees msgid for another
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen message and writes it using a new string index
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen 3) if we still see X, we now see the same msgid with two
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen string indexes.
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen if we detect such a conflict, we can't continue using the
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen strmap index until X has been expunged. */
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen while ((hash_rec = hash2_iterate(ctx->view->hash,
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen /* CRC32 matches, but string index doesn't */
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen if (!strmap_view_sync_handle_conflict(ctx, hash_rec, &iter)) {
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainenmail_index_strmap_view_sync_block(struct mail_index_strmap_read_context *ctx)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while ((ret = strmap_read_block_next(ctx, &crc32)) > 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->rec.uid <= ctx->view->last_added_uid) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->rec.uid < ctx->view->last_added_uid ||
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we've already added this */
39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen if (strmap_view_sync_block_check_conflicts(ctx, crc32) < 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* add the record to records array */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&ctx->view->recs_crc32, &crc32, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* add a separate copy of the record to hash */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash_rec = hash2_insert_hash(ctx->view->hash, crc32);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(hash_rec, &ctx->rec, sizeof(*hash_rec));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return strmap_read_block_deinit(ctx, ret, TRUE);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_view_sync_init(struct mail_index_strmap_view *view,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sync = i_new(struct mail_index_strmap_view_sync, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* reading the strmap failed - just ignore and do
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen this in-memory based on whatever we knew last */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while ((ret = strmap_read_block_init(view, &ctx)) > 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_view_sync_block(&ctx) < 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* something failed - we can still use the strmap as far
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen as we managed to read it, but our view is now out
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic inline uint32_t crc32_str_nonzero(const char *str)
77463499e3a97a9b386e27fb1e6409263726ef5bTimo Sirainen /* we'll flip the bits because of a bug in our old crc32 code.
77463499e3a97a9b386e27fb1e6409263726ef5bTimo Sirainen this keeps the index format backwards compatible with the new fixed
77463499e3a97a9b386e27fb1e6409263726ef5bTimo Sirainen crc32 code. */
77463499e3a97a9b386e27fb1e6409263726ef5bTimo Sirainen uint32_t value = crc32_str(str) ^ 0xffffffffU;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const char *key)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = sync->view;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen old_rec = hash2_lookup(view->hash, &hash_key);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* The string already exists, use the same unique idx */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Newly seen string, assign a new unique idx to it */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&view->recs_crc32, &hash_key.crc32, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_add_unique(struct mail_index_strmap_view_sync *sync,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = sync->view;
b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainenmail_index_strmap_zero_terminate(struct mail_index_strmap_view *view)
b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen /* zero-terminate the records array */
b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen array_delete(&view->recs, array_count(&view->recs)-1, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_view_renumber(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_rec *recs, *hash_rec;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t prev_uid, str_idx, *recs_crc32, *renumber_map;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* create a map of old -> new index and remove records of
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen expunged messages */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen renumber_map = i_new(uint32_t, view->next_str_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs = array_get_modifiable(&view->recs, &count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs_crc32 = array_get_modifiable(&view->recs_crc32, &count2);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* see if this record should be removed */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_uid_exists(&ctx, prev_uid);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* message expunged */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } while (i < count && recs[i].uid == prev_uid);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(recs[i].str_idx < view->next_str_idx);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_delete(&view->recs_crc32, dest, i-dest);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* notify caller of the renumbering */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->remap_cb(renumber_map, view->next_str_idx, str_idx + 1,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* renumber the indexes in-place and recreate the hash */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs = array_get_modifiable(&view->recs, &count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs[i].str_idx = renumber_map[recs[i].str_idx];
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash_rec = hash2_insert_hash(view->hash, recs_crc32[i]);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(hash_rec, &recs[i], sizeof(*hash_rec));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* update the new next_str_idx only after remapping */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainenstatic void mail_index_strmap_write_block(struct mail_index_strmap_view *view,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int j, n, count, count2, uid_rec_count;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint8_t *p, packed[MAIL_INDEX_PACK_MAX_SIZE*2];
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* skip over the block size for now, we don't know it yet */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &block_size, sizeof(block_size));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write records */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen crc32 = array_get(&view->recs_crc32, &count2);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while (i < count) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* @UNSAFE: <uid diff> */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_pack_num(&p, recs[i].uid - base_uid);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* find how many records belong to this UID */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* <n> <crc32>*count <str_idx>*count -
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen FIXME: thread index specific code */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Only Message-ID: header */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* In-Reply-To: header */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* References: header */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (j = 0; j < uid_rec_count; j++)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &crc32[i+j], sizeof(crc32[i+j]));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (j = 0; j < uid_rec_count; j++) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(j < 2 || recs[i+j].ref_index == j+1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we know the block size now - write it */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = output->offset - (block_offset + sizeof(block_size));
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = mail_index_uint32_to_offset(block_size << 2);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &block_size, sizeof(block_size));
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen i_assert(view->last_added_uid == recs[count-1].uid);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen view->last_read_block_offset = output->offset;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_recreate_write(struct mail_index_strmap_view *view,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write header */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_index_strmap_write_block(view, output, 0, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_recreate(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* everything expunged - just unlink the existing index */
b1e7bc754b8be7974aea48cb97c5ce866f9b2029Timo Sirainen if (unlink(strmap->path) < 0 && errno != ENOENT)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "unlink()");
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen fd = safe_mkstemp_hostpid_group(str, view->view->index->mode,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "safe_mkstemp_hostpid(%s) failed: %m",
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_recreate_write(view, output);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (ret == 0 && rename(temp_path, strmap->path) < 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "rename(%s, %s) failed: %m",
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_lock(struct mail_index_strmap *strmap)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen timeout_secs = I_MIN(MAIL_INDEX_STRMAP_TIMEOUT_SECS,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = file_wait_lock(strmap->fd, strmap->path, F_WRLCK,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "file_wait_lock()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = file_dotlock_create(&strmap->dotlock_settings,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "file_dotlock_create()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_unlock(struct mail_index_strmap *strmap)
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainenstrmap_rec_cmp(const uint32_t *uid, const struct mail_index_strmap_rec *rec)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_write_append(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int i, old_count;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Check first if another process had written new records to the file.
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen If there are any, hopefully they're the same as what we would be
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen writing. There are two problematic cases when messages have been
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen expunged recently:
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen 1) The file contains UIDs that we don't have. This means the string
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen indexes won't be compatible anymore, so we'll have to renumber ours
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen to match the ones in the strmap file.
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen Currently we don't bother handling 1) case. If indexes don't match
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen what we have, we just don't write anything.
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen 2) We have UIDs that don't exist in the file. We can't simply skip
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen those records, because other records may have pointers to them using
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen different string indexes than we have. Even if we renumbered those,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen future appends by other processes might cause the same problem (they
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen see the string for the first time and assign it a new index, but we
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen already have internally given it another index). So the only
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sensible choice is to write nothing and hope that the message goes
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen away soon. */
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen (void)array_bsearch_insert_pos(&view->recs, &next_uid,
fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen old_recs = array_get(&view->recs, &old_count);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while (i > 0 && old_recs[i-1].uid == old_recs[i].uid)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_seek(view->strmap->input, view->last_read_block_offset);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (ret = strmap_read_block_init(view, &ctx)) > 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while ((ret = strmap_read_block_next(&ctx, &crc32)) > 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* mismatch */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* 1) case */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* 2) case */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* string index mismatch,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen shouldn't happen */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap_read_block_deinit(&ctx, ret, full_block) < 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* nothing new to write */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(old_recs[i].uid > view->last_read_uid);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write the new records */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen output = o_stream_create_fd(view->strmap->fd, 0, FALSE);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)o_stream_seek(output, view->last_read_block_offset);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_index_strmap_write_block(view, output, i,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(view->strmap, "write()");
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_write(struct mail_index_strmap_view *view)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: this renumbering doesn't work well when running for a long
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen time since records aren't removed from hash often enough */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (STRIDX_MUST_RENUMBER(view->next_str_idx - 1,
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!MAIL_INDEX_IS_IN_MEMORY(view->strmap->index)) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index) || view->desynced)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* initial file creation */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* append the new records to the strmap file */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_lock(view->strmap) <= 0) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* timeout / error */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (mail_index_strmap_need_reopen(view->strmap)) {
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* the file was already recreated - leave the syncing as it is
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for now and let the next sync re-read the file. */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_commit(struct mail_index_strmap_view_sync **_sync)
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view_sync *sync = *_sync;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = sync->view;
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* zero-terminate the records array */
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_delete(&view->recs, array_count(&view->recs)-1, 1);
a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_rollback(struct mail_index_strmap_view_sync **_sync)