a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "lib.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "array.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "bsearch-insert-pos.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "istream.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "ostream.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "file-lock.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "file-dotlock.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "crc32.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "safe-mkstemp.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "str.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "mail-index-private.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include "mail-index-strmap.h"

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#include <stdio.h>

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index *index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen char *path;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int fd;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct istream *input;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct file_lock *file_lock;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct dotlock *dotlock;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct dotlock_settings dotlock_settings;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen};

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap_view {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_view *view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ARRAY_TYPE(mail_index_strmap_rec) recs;

4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ARRAY(uint32_t) recs_crc32;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct hash2_table *hash;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_key_cmp_t *key_compare;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_rec_cmp_t *rec_compare;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_remap_t *remap_cb;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen void *cb_context;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uoff_t last_read_block_offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t last_read_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t last_added_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t total_ref_count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t last_ref_index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t next_str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t lost_expunged_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool desynced:1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen};

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap_read_context {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct istream *input;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uoff_t end_offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t highest_str_idx;

030e247340315f4425d297a00b439a4df72bf70cPhil Carmody uint32_t uid_lookup_seq;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t lost_expunged_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data, *end, *str_idx_base;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_rec rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t next_ref_index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int rec_size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

0dffa25d211be541ee3c953b23566a1a990789dfTimo Sirainen bool too_large_uids:1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen};

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap_view_sync {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen};

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap_hash_key {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const char *str;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t crc32;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen};

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#define STRMAP_FILE_STRIDX_SIZE (sizeof(uint32_t)*2)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#define STRMAP_FILE_MAX_STRIDX_MULTIPLIER 2

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#define STRIDX_MUST_RENUMBER(highest_idx, n_unique_indexes) \

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (highest_idx > n_unique_indexes * STRMAP_FILE_MAX_STRIDX_MULTIPLIER)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen#define MAIL_INDEX_STRMAP_TIMEOUT_SECS 10

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

0b3e92b6043435c5aa9f1cf1d04b632f3e19abd9Phil Carmodystatic const struct dotlock_settings default_dotlock_settings = {

7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen .timeout = MAIL_INDEX_STRMAP_TIMEOUT_SECS,

7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen .stale_timeout = 30

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen};

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap *

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_init(struct mail_index *index, const char *suffix)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

54e5dadb95d8b7ffce6b5a1f6dbba22044663451Timo Sirainen i_assert(index->open_count > 0);

54e5dadb95d8b7ffce6b5a1f6dbba22044663451Timo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap = i_new(struct mail_index_strmap, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->index = index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->path = i_strconcat(index->filepath, suffix, NULL);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->fd = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->dotlock_settings = default_dotlock_settings;

66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen strmap->dotlock_settings.use_excl_lock =

66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;

66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen strmap->dotlock_settings.nfs_flush =

66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic bool

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t *crc32_r);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_set_syscall_error(struct mail_index_strmap *strmap,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const char *function)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(function != NULL);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ENOSPACE(errno)) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->index->nodiskspace = TRUE;

66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen if ((strmap->index->flags &

66ecc94150cbce23aad3240135e0782e0a74d479Timo Sirainen MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) == 0)

ef5fb27361cc5e15766e85e28355750ff04b13c9Timo Sirainen return;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_set_error(strmap->index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "%s failed with strmap index file %s: %m",

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen function, strmap->path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_close(struct mail_index_strmap *strmap)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->file_lock != NULL)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen file_lock_free(&strmap->file_lock);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen else if (strmap->dotlock != NULL)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen file_dotlock_delete(&strmap->dotlock);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->fd != -1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (close(strmap->fd) < 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "close()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->fd = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

204ee6ed414f5e4eeb6f6c10763b55daf56f11acJosef 'Jeff' Sipek i_stream_unref(&strmap->input);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_deinit(struct mail_index_strmap **_strmap)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = *_strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen *_strmap = NULL;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_close(strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_free(strmap->path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_free(strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic unsigned int mail_index_strmap_hash_key(const void *_key)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_hash_key *key = _key;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return key->crc32;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic bool

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_hash_cmp(const void *_key, const void *_value, void *context)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_hash_key *key = _key;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_rec *rec = _value;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = context;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return view->key_compare(key->str, rec, view->cb_context);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap_view *

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_view_open(struct mail_index_strmap *strmap,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_view *idx_view,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_key_cmp_t *key_compare_cb,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_rec_cmp_t *rec_compare_cb,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_remap_t *remap_cb,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen void *context,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const ARRAY_TYPE(mail_index_strmap_rec) **recs_r,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct hash2_table **hash_r)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view = i_new(struct mail_index_strmap_view, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->strmap = strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->view = idx_view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->key_compare = key_compare_cb;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->rec_compare = rec_compare_cb;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->remap_cb = remap_cb;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->cb_context = context;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->next_str_idx = 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_array_init(&view->recs, 64);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_array_init(&view->recs_crc32, 64);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->hash = hash2_create(0, sizeof(struct mail_index_strmap_rec),

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_hash_key,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_hash_cmp, view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen *recs_r = &view->recs;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen *hash_r = view->hash;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_close(struct mail_index_strmap_view **_view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = *_view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen *_view = NULL;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_free(&view->recs);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_free(&view->recs_crc32);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash2_destroy(&view->hash);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_free(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenuint32_t mail_index_strmap_view_get_highest_idx(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return view->next_str_idx-1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_view_reset(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->remap_cb(NULL, 0, 0, view->cb_context);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_clear(&view->recs);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_clear(&view->recs_crc32);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash2_clear(view->hash);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_added_uid = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->lost_expunged_uid = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->desynced = FALSE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

f24202d1a342ac63fb8c28e6fa74e16d37f29ca3Timo Sirainenvoid mail_index_strmap_view_set_corrupted(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_set_error(view->strmap->index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Corrupted strmap index file: %s",

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->strmap->path);

46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen i_unlink(view->strmap->path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_close(view->strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_reset(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_open(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_header *idx_hdr;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_header hdr;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size_t size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(strmap->fd == -1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->fd = open(strmap->path, O_RDWR);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->fd == -1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (errno == ENOENT)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "open()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

e93184a9055c2530366dfe617e07199603c399ddMartti Rannanjärvi strmap->input = i_stream_create_fd(strmap->fd, (size_t)-1);

534e41e436a7546f36f61e0dc49c0c191d850f6bPhil Carmody ret = i_stream_read_bytes(strmap->input, &data, &size, sizeof(hdr));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret <= 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "read()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_close(strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(ret == 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_set_corrupted(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&hdr, data, sizeof(hdr));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen idx_hdr = mail_index_get_header(view->view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (hdr.version != MAIL_INDEX_STRMAP_VERSION ||

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hdr.uid_validity != idx_hdr->uid_validity) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* need to rebuild. if we already had something in the strmap,

46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen i_unlink(strmap->path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_close(strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we'll read the entire file from the beginning */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_added_uid = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_uid = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->total_ref_count = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_block_offset = sizeof(struct mail_index_strmap_header);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->next_str_idx = 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_reset(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic bool mail_index_strmap_need_reopen(struct mail_index_strmap *strmap)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct stat st1, st2;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: nfs flush */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (fstat(strmap->fd, &st1) < 0) {

85da8c055280cd45553b6b335e9fb226d6e2801eTimo Sirainen if (!ESTALE_FSTAT(errno))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "fstat()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (stat(strmap->path, &st2) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "stat()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return st1.st_ino != st2.st_ino || !CMP_DEV_T(st1.st_dev, st2.st_dev);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_refresh(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t seq;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->strmap->fd != -1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_strmap_need_reopen(view->strmap)) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->lost_expunged_uid != 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* last read failed because view had a message

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_lookup_seq(view->view,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->lost_expunged_uid, &seq))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (view->desynced) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* our view isn't synced with the disk, we

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_sync(view->strmap->input);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_close(view->strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return mail_index_strmap_open(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_packed(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t *num_r)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const uint8_t *bytes, *p, *end;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size_t size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

534e41e436a7546f36f61e0dc49c0c191d850f6bPhil Carmody ret = i_stream_read_bytes(ctx->input, &data, &size, sizeof(*num_r));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret <= 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->input->v_offset + size > ctx->end_offset)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size = ctx->end_offset - ctx->input->v_offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen bytes = p = (const uint8_t *)data;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen end = bytes + size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_unpack_num(&p, end, num_r) < 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(ctx->input, p - bytes);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_uid_exists(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t uid)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_record *rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

00cc12d660b35bc955f97d30af660f3879b35a3cTimo Sirainen i_assert(ctx->uid_lookup_seq > 0);

00cc12d660b35bc955f97d30af660f3879b35a3cTimo Sirainen

030e247340315f4425d297a00b439a4df72bf70cPhil Carmody if (ctx->uid_lookup_seq > ctx->view->view->map->hdr.messages_count) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (uid >= ctx->view->view->map->hdr.next_uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* thread index has larger UIDs than what we've seen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->too_large_uids = TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

030e247340315f4425d297a00b439a4df72bf70cPhil Carmody rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->view->map, ctx->uid_lookup_seq);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (rec->uid == uid) {

030e247340315f4425d297a00b439a4df72bf70cPhil Carmody ctx->uid_lookup_seq++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (rec->uid > uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* record that exists in index is missing from strmap.

b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen mail_index_refresh(ctx->view->view->index);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_is_expunged(ctx->view->view,

030e247340315f4425d297a00b439a4df72bf70cPhil Carmody ctx->uid_lookup_seq))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->lost_expunged_uid = rec->uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_rec_first(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t *crc32_r)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size_t size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t n, i, count, str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* <uid> <n> <crc32>*count <str_idx>*count

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_read_packed(ctx, &n) <= 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen count = n < 2 ? n + 1 : n;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->view->total_ref_count += count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->rec_size = count * (sizeof(ctx->rec.str_idx) + sizeof(*crc32_r));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_uid_exists(ctx, ctx->rec.uid);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret < 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

573424407a2d3c1453638a643583a7cf10c129e1Phil Carmody if (i_stream_read_bytes(ctx->view->strmap->input, &ctx->data, &size, ctx->rec_size) <= 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->str_idx_base = ctx->data + count * sizeof(uint32_t);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret == 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* this message has already been expunged, ignore it.

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&str_idx, ctx->str_idx_base, sizeof(str_idx));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->highest_str_idx < str_idx)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->highest_str_idx = str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->str_idx_base += sizeof(str_idx);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(ctx->view->strmap->input, ctx->rec_size);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* everything exists. save it. FIXME: these ref_index values

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->end = ctx->data + count * sizeof(*crc32_r);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->next_ref_index = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_strmap_read_rec_next(ctx, crc32_r))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_unreached();

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->next_ref_index = n == 1 ? 1 : 2;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic bool

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t *crc32_r)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->data == ctx->end) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(ctx->view->strmap->input, ctx->rec_size);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return FALSE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: str_idx could be stored as packed relative values

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* read the record contents */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&ctx->rec.str_idx, ctx->str_idx_base, sizeof(ctx->rec.str_idx));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(crc32_r, ctx->data, sizeof(*crc32_r));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->rec.ref_index = ctx->next_ref_index++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->highest_str_idx < ctx->rec.str_idx)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->highest_str_idx = ctx->rec.str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* get to the next record */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->data += sizeof(*crc32_r);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->str_idx_base += sizeof(ctx->rec.str_idx);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_read_block_init(struct mail_index_strmap_view *view,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_read_context *ctx)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const unsigned char *data;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen size_t size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t block_size, seq1, seq2;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->last_read_uid + 1 >= view->view->map->hdr.next_uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* come back later when we know about the new UIDs */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(ctx);

573424407a2d3c1453638a643583a7cf10c129e1Phil Carmody ret = i_stream_read_bytes(strmap->input, &data, &size,

573424407a2d3c1453638a643583a7cf10c129e1Phil Carmody sizeof(block_size));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret <= 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->input->stream_errno == 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* no new data */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "read()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(&block_size, data, sizeof(block_size));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = mail_index_offset_to_uint32(block_size) >> 2;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (block_size == 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* the rest of the file is either not written, or the previous

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_skip(strmap->input, sizeof(block_size));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->view = view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->input = strmap->input;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->end_offset = strmap->input->v_offset + block_size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->end_offset < strmap->input->v_offset) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* block size too large */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_set_corrupted(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->rec.uid = view->last_read_uid + 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: when reading multiple blocks we shouldn't have to calculate

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_lookup_seq_range(view->view, ctx->rec.uid, (uint32_t)-1,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen &seq1, &seq2))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen seq1 = mail_index_view_get_messages_count(view->view) + 1;

030e247340315f4425d297a00b439a4df72bf70cPhil Carmody ctx->uid_lookup_seq = seq1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_read_block_next(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t *crc32_r)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t uid_diff;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_read_rec_next(ctx, crc32_r))

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* get next UID */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen do {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->input->v_offset == ctx->end_offset) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* this block is done */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

ac4f179bd60c1738cad4ba10f825a67496d3f715Teemu Huovila if (mail_index_strmap_read_packed(ctx, &uid_diff) <= 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->rec.uid += uid_diff;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_read_rec_first(ctx, crc32_r);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } while (ret == 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_read_block_deinit(struct mail_index_strmap_read_context *ctx, int ret,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen bool update_block_offset)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = ctx->view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->highest_str_idx > view->total_ref_count) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* if all string indexes are unique, highest_str_index equals

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_set_error(strmap->index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "Corrupted strmap index file %s: "

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "String indexes too high "

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "(highest=%u max=%u)",

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->path, ctx->highest_str_idx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->total_ref_count);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_set_corrupted(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->lost_expunged_uid != 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* our view contained a message that had since been expunged. */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(ret < 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->lost_expunged_uid = ctx->lost_expunged_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (ret < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->input->stream_errno != 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "read()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen else

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_set_corrupted(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (update_block_offset && !ctx->too_large_uids) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_block_offset = strmap->input->v_offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_uid = ctx->rec.uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->next_str_idx <= ctx->highest_str_idx)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->next_str_idx = ctx->highest_str_idx + 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic bool

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstrmap_view_sync_handle_conflict(struct mail_index_strmap_read_context *ctx,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_rec *hash_rec,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct hash2_iter *iter)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t seq;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* hopefully it's a message that has since been expunged */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!mail_index_lookup_seq(ctx->view->view, hash_rec->uid, &seq)) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* message is no longer in our view. remove it completely. */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash2_remove_iter(ctx->view->hash, iter);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_is_expunged(ctx->view->view, seq)) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* it's quite likely a conflict. we may not be able to verify

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return FALSE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* 0 means "doesn't match", which is the only acceptable case */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ctx->view->rec_compare(&ctx->rec, hash_rec,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->view->cb_context) == 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainenstrmap_view_sync_block_check_conflicts(struct mail_index_strmap_read_context *ctx,

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen uint32_t crc32)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_rec *hash_rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct hash2_iter iter;

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen if (crc32 == 0) {

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen /* unique string - there are no conflicts */

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen return 0;

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen }

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen /* check for conflicting string indexes. they may happen if

efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&iter);

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen while ((hash_rec = hash2_iterate(ctx->view->hash,

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen crc32, &iter)) != NULL &&

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen hash_rec->str_idx != ctx->rec.str_idx) {

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen /* CRC32 matches, but string index doesn't */

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen if (!strmap_view_sync_handle_conflict(ctx, hash_rec, &iter)) {

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen ctx->lost_expunged_uid = hash_rec->uid;

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen return -1;

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen }

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen }

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen return 0;

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen}

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainenstatic int

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainenmail_index_strmap_view_sync_block(struct mail_index_strmap_read_context *ctx)

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen{

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen struct mail_index_strmap_rec *hash_rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t crc32, prev_uid = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while ((ret = strmap_read_block_next(ctx, &crc32)) > 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->rec.uid <= ctx->view->last_added_uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx->rec.uid < ctx->view->last_added_uid ||

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen prev_uid != ctx->rec.uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we've already added this */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen continue;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen prev_uid = ctx->rec.uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen if (strmap_view_sync_block_check_conflicts(ctx, crc32) < 0) {

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen ret = -1;

39413f6b07f7e4f4c1aeeecab73a2c454c84e308Timo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx->view->last_added_uid = ctx->rec.uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* add the record to records array */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&ctx->view->recs, &ctx->rec, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&ctx->view->recs_crc32, &crc32, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* add a separate copy of the record to hash */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash_rec = hash2_insert_hash(ctx->view->hash, crc32);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(hash_rec, &ctx->rec, sizeof(*hash_rec));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return strmap_read_block_deinit(ctx, ret, TRUE);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstruct mail_index_strmap_view_sync *

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_view_sync_init(struct mail_index_strmap_view *view,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t *last_uid_r)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view_sync *sync;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_read_context ctx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sync = i_new(struct mail_index_strmap_view_sync, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen sync->view = view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_refresh(view) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* reading the strmap failed - just ignore and do

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (view->strmap->input != NULL) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_seek(view->strmap->input,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_read_block_offset);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while ((ret = strmap_read_block_init(view, &ctx)) > 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_view_sync_block(&ctx) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx.too_large_uids)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* something failed - we can still use the strmap as far

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->desynced = TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(view->lost_expunged_uid == 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen *last_uid_r = view->last_added_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return sync;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic inline uint32_t crc32_str_nonzero(const char *str)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

77463499e3a97a9b386e27fb1e6409263726ef5bTimo Sirainen /* we'll flip the bits because of a bug in our old crc32 code.

77463499e3a97a9b386e27fb1e6409263726ef5bTimo Sirainen uint32_t value = crc32_str(str) ^ 0xffffffffU;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return value == 0 ? 1 : value;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t uid, uint32_t ref_index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const char *key)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = sync->view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_rec *rec, *old_rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_hash_key hash_key;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(uid > view->last_added_uid ||

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (uid == view->last_added_uid &&

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ref_index > view->last_ref_index));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash_key.str = key;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash_key.crc32 = crc32_str_nonzero(key);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen old_rec = hash2_lookup(view->hash, &hash_key);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (old_rec != NULL) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* The string already exists, use the same unique idx */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen str_idx = old_rec->str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Newly seen string, assign a new unique idx to it */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen str_idx = view->next_str_idx++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(str_idx != 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec = hash2_insert(view->hash, &hash_key);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec->uid = uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec->ref_index = ref_index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec->str_idx = str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&view->recs, rec, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&view->recs_crc32, &hash_key.crc32, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_added_uid = uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_ref_index = ref_index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_add_unique(struct mail_index_strmap_view_sync *sync,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t uid, uint32_t ref_index)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = sync->view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_rec rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(uid > view->last_added_uid ||

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (uid == view->last_added_uid &&

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ref_index > view->last_ref_index));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&rec);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec.uid = uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec.ref_index = ref_index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen rec.str_idx = view->next_str_idx++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_append(&view->recs, &rec, 1);

31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&view->recs_crc32);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_added_uid = uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->last_ref_index = ref_index;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainenstatic void

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainenmail_index_strmap_zero_terminate(struct mail_index_strmap_view *view)

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen{

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen /* zero-terminate the records array */

31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen array_append_zero(&view->recs);

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen array_delete(&view->recs, array_count(&view->recs)-1, 1);

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen}

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_view_renumber(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_read_context ctx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_rec *recs, *hash_rec;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t prev_uid, str_idx, *recs_crc32, *renumber_map;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int i, dest, count, count2;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&ctx);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx.view = view;

00cc12d660b35bc955f97d30af660f3879b35a3cTimo Sirainen ctx.uid_lookup_seq = 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* create a map of old -> new index and remove records of

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen renumber_map = i_new(uint32_t, view->next_str_idx);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen str_idx = 0; prev_uid = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs = array_get_modifiable(&view->recs, &count);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs_crc32 = array_get_modifiable(&view->recs_crc32, &count2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(count == count2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = dest = 0; i < count; ) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (prev_uid != recs[i].uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* see if this record should be removed */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen prev_uid = recs[i].uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_uid_exists(&ctx, prev_uid);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(ret >= 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret == 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* message expunged */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen do {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } while (i < count && recs[i].uid == prev_uid);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen continue;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(recs[i].str_idx < view->next_str_idx);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (renumber_map[recs[i].str_idx] == 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen renumber_map[recs[i].str_idx] = ++str_idx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (i != dest) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs[dest] = recs[i];

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs_crc32[dest] = recs_crc32[i];

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i++; dest++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(renumber_map[0] == 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_delete(&view->recs, dest, i-dest);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen array_delete(&view->recs_crc32, dest, i-dest);

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen mail_index_strmap_zero_terminate(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* notify caller of the renumbering */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(str_idx <= view->next_str_idx);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->remap_cb(renumber_map, view->next_str_idx, str_idx + 1,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->cb_context);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* renumber the indexes in-place and recreate the hash */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs = array_get_modifiable(&view->recs, &count);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash2_clear(view->hash);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (i = 0; i < count; i++) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs[i].str_idx = renumber_map[recs[i].str_idx];

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash_rec = hash2_insert_hash(view->hash, recs_crc32[i]);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen memcpy(hash_rec, &recs[i], sizeof(*hash_rec));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* update the new next_str_idx only after remapping */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->next_str_idx = str_idx + 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_free(renumber_map);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainenstatic void mail_index_strmap_write_block(struct mail_index_strmap_view *view,

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen struct ostream *output,

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen unsigned int i, uint32_t base_uid)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_rec *recs;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const uint32_t *crc32;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int j, n, count, count2, uid_rec_count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t block_size;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint8_t *p, packed[MAIL_INDEX_PACK_MAX_SIZE*2];

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uoff_t block_offset, end_offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* skip over the block size for now, we don't know it yet */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_offset = output->offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = 0;

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &block_size, sizeof(block_size));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write records */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen recs = array_get(&view->recs, &count);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen crc32 = array_get(&view->recs_crc32, &count2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(count == count2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while (i < count) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* @UNSAFE: <uid diff> */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen p = packed;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_pack_num(&p, recs[i].uid - base_uid);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen base_uid = recs[i].uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* find how many records belong to this UID */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uid_rec_count = 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (j = i + 1; j < count; j++) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (recs[j].uid != base_uid)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uid_rec_count++;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->total_ref_count += uid_rec_count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* <n> <crc32>*count <str_idx>*count -

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(recs[i].ref_index == 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (uid_rec_count == 1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Only Message-ID: header */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen n = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (recs[i+1].ref_index == 1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* In-Reply-To: header */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen n = 1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(uid_rec_count == 2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* References: header */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen n = uid_rec_count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(recs[i+1].ref_index == 2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_pack_num(&p, n);

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, packed, p-packed);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (j = 0; j < uid_rec_count; j++)

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &crc32[i+j], sizeof(crc32[i+j]));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen for (j = 0; j < uid_rec_count; j++) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(j < 2 || recs[i+j].ref_index == j+1);

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &recs[i+j].str_idx,

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen sizeof(recs[i+j].str_idx));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i += uid_rec_count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* we know the block size now - write it */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = output->offset - (block_offset + sizeof(block_size));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen block_size = mail_index_uint32_to_offset(block_size << 2);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(block_size != 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen end_offset = output->offset;

e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)o_stream_seek(output, block_offset);

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &block_size, sizeof(block_size));

e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)o_stream_seek(output, end_offset);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

f988b93c2ef773987bcdcbfb4cca39b955e3a392Timo Sirainen if (output->stream_errno != 0)

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return;

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen i_assert(view->last_added_uid == recs[count-1].uid);

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen view->last_read_uid = recs[count-1].uid;

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen view->last_read_block_offset = output->offset;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_recreate_write(struct mail_index_strmap_view *view,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct ostream *output)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_header *idx_hdr;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_header hdr;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen idx_hdr = mail_index_get_header(view->view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write header */

efe78d3ba24fc866af1c79b9223dc0809ba26cadStephan Bosch i_zero(&hdr);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hdr.version = MAIL_INDEX_STRMAP_VERSION;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hdr.uid_validity = idx_hdr->uid_validity;

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen o_stream_nsend(output, &hdr, sizeof(hdr));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->total_ref_count = 0;

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_index_strmap_write_block(view, output, 0, 1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_recreate(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap *strmap = view->strmap;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen string_t *str;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct ostream *output;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const char *temp_path;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int fd, ret = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (array_count(&view->recs) == 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* everything expunged - just unlink the existing index */

b1e7bc754b8be7974aea48cb97c5ce866f9b2029Timo Sirainen if (unlink(strmap->path) < 0 && errno != ENOENT)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap, "unlink()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen str = t_str_new(256);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen str_append(str, strmap->path);

e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen fd = safe_mkstemp_hostpid_group(str, view->view->index->mode,

e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen view->view->index->gid,

e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen view->view->index->gid_origin);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen temp_path = str_c(str);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (fd == -1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_set_error(strmap->index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "safe_mkstemp_hostpid(%s) failed: %m",

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen temp_path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

e93184a9055c2530366dfe617e07199603c399ddMartti Rannanjärvi output = o_stream_create_fd(fd, 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen o_stream_cork(output);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_recreate_write(view, output);

ad9afb64630511d5e25bc5bc11c5304986156928Timo Sirainen if (o_stream_finish(output) < 0) {

0f3d4fbcf88e2ffd674893aed8cc1288fe17d290Timo Sirainen mail_index_set_error(strmap->index, "write(%s) failed: %s",

0f3d4fbcf88e2ffd674893aed8cc1288fe17d290Timo Sirainen temp_path, o_stream_get_error(output));

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen o_stream_destroy(&output);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (close(fd) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_set_error(strmap->index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "close(%s) failed: %m", temp_path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (ret == 0 && rename(temp_path, strmap->path) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_set_error(strmap->index,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "rename(%s, %s) failed: %m",

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen temp_path, strmap->path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret < 0)

46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen i_unlink(temp_path);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_lock(struct mail_index_strmap *strmap)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen unsigned int timeout_secs;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(strmap->fd != -1);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(strmap->file_lock == NULL);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen timeout_secs = I_MIN(MAIL_INDEX_STRMAP_TIMEOUT_SECS,

036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen strmap->index->max_lock_timeout_secs);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = file_wait_lock(strmap->fd, strmap->path, F_WRLCK,

036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen strmap->index->lock_method, timeout_secs,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen &strmap->file_lock);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret <= 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "file_wait_lock()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(strmap->dotlock == NULL);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = file_dotlock_create(&strmap->dotlock_settings,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen strmap->path, 0, &strmap->dotlock);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret <= 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(strmap,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen "file_dotlock_create()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic void mail_index_strmap_unlock(struct mail_index_strmap *strmap)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap->file_lock != NULL)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen file_unlock(&strmap->file_lock);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen else if (strmap->dotlock != NULL)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen file_dotlock_delete(&strmap->dotlock);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainenstatic int

fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainenstrmap_rec_cmp(const uint32_t *uid, const struct mail_index_strmap_rec *rec)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return *uid < rec->uid ? -1 :

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (*uid > rec->uid ? 1 : 0);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenmail_index_strmap_write_append(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_read_context ctx;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen const struct mail_index_strmap_rec *old_recs;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen unsigned int i, old_count;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct ostream *output;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen uint32_t crc32, next_uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen bool full_block;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* Check first if another process had written new records to the file.

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen next_uid = view->last_read_uid + 1;

fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen (void)array_bsearch_insert_pos(&view->recs, &next_uid,

fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen strmap_rec_cmp, &i);

fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen

fde0b1793a2842da00eaa105d5e13fec465f0443Timo Sirainen old_recs = array_get(&view->recs, &old_count);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (i < old_count) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while (i > 0 && old_recs[i-1].uid == old_recs[i].uid)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i--;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_sync(view->strmap->input);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_stream_seek(view->strmap->input, view->last_read_block_offset);

49f8bdc504635afec3016a6973aac06d6ff8403fTimo Sirainen full_block = TRUE; ret = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while (i < old_count &&

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (ret = strmap_read_block_init(view, &ctx)) > 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen while ((ret = strmap_read_block_next(&ctx, &crc32)) > 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx.rec.uid != old_recs[i].uid ||

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx.rec.str_idx != old_recs[i].str_idx) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* mismatch */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ctx.rec.uid > old_recs[i].uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* 1) case */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx.lost_expunged_uid = ctx.rec.uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (ctx.rec.uid < old_recs[i].uid) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* 2) case */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ctx.lost_expunged_uid = old_recs[i].uid;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* string index mismatch,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (++i == old_count) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen full_block = FALSE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (strmap_read_block_deinit(&ctx, ret, full_block) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen break;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret < 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (i == old_count) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* nothing new to write */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(full_block);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_assert(old_recs[i].uid > view->last_read_uid);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* write the new records */

e93184a9055c2530366dfe617e07199603c399ddMartti Rannanjärvi output = o_stream_create_fd(view->strmap->fd, 0);

e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen (void)o_stream_seek(output, view->last_read_block_offset);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen o_stream_cork(output);

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_index_strmap_write_block(view, output, i,

e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen view->last_read_uid + 1);

ad9afb64630511d5e25bc5bc11c5304986156928Timo Sirainen if (o_stream_finish(output) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_set_syscall_error(view->strmap, "write()");

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen o_stream_destroy(&output);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenstatic int mail_index_strmap_write(struct mail_index_strmap_view *view)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen int ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* FIXME: this renumbering doesn't work well when running for a long

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (STRIDX_MUST_RENUMBER(view->next_str_idx - 1,

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen hash2_count(view->hash))) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_view_renumber(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (!MAIL_INDEX_IS_IN_MEMORY(view->strmap->index)) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_recreate(view) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->desynced = TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index) || view->desynced)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (view->strmap->fd == -1) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* initial file creation */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_recreate(view) < 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->desynced = TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* append the new records to the strmap file */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (mail_index_strmap_lock(view->strmap) <= 0) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* timeout / error */

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = -1;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else if (mail_index_strmap_need_reopen(view->strmap)) {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen /* the file was already recreated - leave the syncing as it is

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = 0;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen } else {

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen ret = mail_index_strmap_write_append(view);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen }

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen mail_index_strmap_unlock(view->strmap);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen if (ret < 0)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen view->desynced = TRUE;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen return ret;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen}

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainenvoid mail_index_strmap_view_sync_commit(struct mail_index_strmap_view_sync **_sync)

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen{

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view_sync *sync = *_sync;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen struct mail_index_strmap_view *view = sync->view;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen *_sync = NULL;

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen i_free(sync);

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen

a914bff43644dd9b3977244203839ca74161e40cTimo Sirainen (void)mail_index_strmap_write(view);

b6dff2ba7a4640c1c4fa8fcad5602d236c31a2e4Timo Sirainen mail_index_strmap_zero_terminate(view);