daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "lib.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "array.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "hash.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "istream.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "ostream.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "str.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "file-dotlock.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "close-keep-errno.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "nfs-workarounds.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "maildir-storage.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "maildir-sync.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "maildir-filename.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa#include "maildir-uidlist.h"

f0264afd33a980b6584747fc8159ee950805d9e3Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa#include <stdio.h>

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include <stdlib.h>

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#include <sys/stat.h>

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#define UIDLIST_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa

d89f470f7da0b9f8295d0ac0defff09884894b8bEugen Kuksa#define UIDLIST_LOCK_STALE_TIMEOUT (60*2)

d89f470f7da0b9f8295d0ac0defff09884894b8bEugen Kuksa

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa#define UIDLIST_COMPRESS_PERCENTAGE 75

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa#define UIDLIST_IS_LOCKED(uidlist) \

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa ((uidlist)->lock_count > 0)

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastruct maildir_uidlist_rec {

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa uint32_t uid;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uint32_t flags;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa char *filename;

67daeb8066e4460cb820db60c45138dd48309bb9Eugen Kuksa unsigned char *extensions; /* <data>\0[<data>\0 ...]\0 */

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa};

daf3e28fff47a65b53d6fb65155301763b9f166eEugen KuksaARRAY_DEFINE_TYPE(maildir_uidlist_rec_p, struct maildir_uidlist_rec *);

57c81a32dff6182f040b4f852892144641a62fe5Eugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastruct maildir_uidlist {

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct maildir_mailbox *mbox;

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa struct index_mailbox *ibox;

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa char *path;

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa int fd;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa dev_t fd_dev;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa ino_t fd_ino;

67daeb8066e4460cb820db60c45138dd48309bb9Eugen Kuksa off_t fd_size;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int lock_count;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa struct dotlock_settings dotlock_settings;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct dotlock *dotlock;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa pool_t record_pool;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa ARRAY_TYPE(maildir_uidlist_rec_p) records;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct hash_table *files;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int change_counter;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa unsigned int version;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int uid_validity, next_uid, prev_read_uid, last_seen_uid;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int read_records_count, read_line_count;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uoff_t last_read_offset;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa string_t *hdr_extensions;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int recreate:1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int initial_read:1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int initial_sync:1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa};

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastruct maildir_uidlist_sync_ctx {

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct maildir_uidlist *uidlist;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa enum maildir_uidlist_sync_flags sync_flags;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa

923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa pool_t record_pool;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa ARRAY_TYPE(maildir_uidlist_rec_p) records;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa struct hash_table *files;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int first_unwritten_pos, first_nouid_pos;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int new_files_count;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int finish_change_counter;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int partial:1;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int finished:1;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int changed:1;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int failed:1;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa unsigned int locked:1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa};

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastruct maildir_uidlist_iter_ctx {

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct maildir_uidlist *uidlist;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct maildir_uidlist_rec *const *next, *const *end;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa unsigned int change_counter;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uint32_t prev_uid;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa};

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastatic bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx,

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa struct maildir_uidlist_rec **rec_r);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksastatic int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist,

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa bool nonblock)

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa{

96c47a92d836bca1aca373db8664c5ec84cfc8afEugen Kuksa struct mailbox *box = &uidlist->ibox->box;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa const char *control_dir, *path;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa mode_t old_mask;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa const enum dotlock_create_flags dotlock_flags =

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0;

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa int i, ret;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (uidlist->lock_count > 0) {

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa uidlist->lock_count++;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return 1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa control_dir = mailbox_list_get_path(box->storage->list, box->name,

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa MAILBOX_LIST_PATH_TYPE_CONTROL);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa path = t_strconcat(control_dir, "/" MAILDIR_UIDLIST_NAME, NULL);

4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa

4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa for (i = 0;; i++) {

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa old_mask = umask(0777 & ~box->file_create_mode);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa ret = file_dotlock_create(&uidlist->dotlock_settings, path,

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa dotlock_flags, &uidlist->dotlock);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa umask(old_mask);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (ret > 0)

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa break;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

f9e467b8fe6fef705eec2989b20e92eaa9d917e1Eugen Kuksa /* failure */

f9e467b8fe6fef705eec2989b20e92eaa9d917e1Eugen Kuksa if (ret == 0) {

f9e467b8fe6fef705eec2989b20e92eaa9d917e1Eugen Kuksa mail_storage_set_error(box->storage,

f9e467b8fe6fef705eec2989b20e92eaa9d917e1Eugen Kuksa MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return 0;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT ||

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uidlist->mbox == NULL) {

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa mail_storage_set_critical(box->storage,

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa "file_dotlock_create(%s) failed: %m", path);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return -1;

2dbc668d1e44c95db1857d3968bcde7517852beaEugen Kuksa }

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa /* the control dir doesn't exist. create it unless the whole

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (maildir_set_deleted(uidlist->mbox))

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return -1;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa }

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa uidlist->lock_count++;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa /* make sure we have the latest changes before changing anything */

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (maildir_uidlist_refresh(uidlist) < 0) {

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa maildir_uidlist_unlock(uidlist);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return -1;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa }

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa return 1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksaint maildir_uidlist_lock(struct maildir_uidlist *uidlist)

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa{

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa return maildir_uidlist_lock_timeout(uidlist, FALSE);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksaint maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa{

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa return maildir_uidlist_lock_timeout(uidlist, TRUE);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksaint maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist)

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa{

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa i_assert(UIDLIST_IS_LOCKED(uidlist));

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return file_dotlock_touch(uidlist->dotlock);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

284432981d641cf3d679841f75acbcf039d83062Eugen Kuksabool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist)

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa{

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return UIDLIST_IS_LOCKED(uidlist);

6cdc461aada609d57d50ff675d29b15378717ff2Eugen Kuksa}

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksavoid maildir_uidlist_unlock(struct maildir_uidlist *uidlist)

76f98a73426869ab54073075345686d98b6a8cafEugen Kuksa{

a8028fd2789e323040de08827a0fe1f7d36fde2bEugen Kuksa i_assert(uidlist->lock_count > 0);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (--uidlist->lock_count > 0)

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa (void)file_dotlock_delete(&uidlist->dotlock);

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksastruct maildir_uidlist *

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksamaildir_uidlist_init_readonly(struct index_mailbox *ibox)

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa{

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa struct mailbox *box = &ibox->box;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa struct maildir_uidlist *uidlist;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa const char *control_dir;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa control_dir = mailbox_list_get_path(box->storage->list, box->name,

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa MAILBOX_LIST_PATH_TYPE_CONTROL);

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa

bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa uidlist = i_new(struct maildir_uidlist, 1);

bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa uidlist->fd = -1;

bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa uidlist->ibox = ibox;

bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa uidlist->path = i_strconcat(control_dir, "/"MAILDIR_UIDLIST_NAME, NULL);

bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa i_array_init(&uidlist->records, 128);

bacf95fddc53ab3107f380c4c816fa8072358bd9Eugen Kuksa uidlist->files = hash_create(default_pool, default_pool, 4096,

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa maildir_filename_base_hash,

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa maildir_filename_base_cmp);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uidlist->next_uid = 1;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uidlist->hdr_extensions = str_new(default_pool, 128);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

4ec9d8b62c3c1a001548eb0883b6f81e00c391a0Eugen Kuksa uidlist->dotlock_settings.use_io_notify = TRUE;

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa uidlist->dotlock_settings.use_excl_lock =

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa (box->storage->flags & MAIL_STORAGE_FLAG_DOTLOCK_USE_EXCL) != 0;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uidlist->dotlock_settings.nfs_flush =

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa (box->storage->flags &

96c47a92d836bca1aca373db8664c5ec84cfc8afEugen Kuksa MAIL_STORAGE_FLAG_NFS_FLUSH_STORAGE) != 0;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa uidlist->dotlock_settings.timeout = UIDLIST_LOCK_STALE_TIMEOUT + 2;

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa uidlist->dotlock_settings.stale_timeout = UIDLIST_LOCK_STALE_TIMEOUT;

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa

96c47a92d836bca1aca373db8664c5ec84cfc8afEugen Kuksa return uidlist;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksastruct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox)

102ec428bcf83c38bdf85922bcd330653eb4c8dcEugen Kuksa{

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa struct maildir_uidlist *uidlist;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa uidlist = maildir_uidlist_init_readonly(&mbox->ibox);

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa uidlist->mbox = mbox;

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa uidlist->dotlock_settings.temp_prefix = mbox->storage->temp_prefix;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa return uidlist;

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa}

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksastatic void maildir_uidlist_close(struct maildir_uidlist *uidlist)

28001d576e67ba46ed481c5695f1e0827ff26007Eugen Kuksa{

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (uidlist->fd != -1) {

b51057b860560bf3ee454c03a121af3d5d34f482Eugen Kuksa if (close(uidlist->fd) < 0)

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa i_error("close(%s) failed: %m", uidlist->path);

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa uidlist->fd = -1;

486df98bbf3348cfb96e93c3e499d12435880bb5Eugen Kuksa uidlist->fd_ino = 0;

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa }

6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa uidlist->last_read_offset = 0;

57c81a32dff6182f040b4f852892144641a62fe5Eugen Kuksa uidlist->read_line_count = 0;

6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa}

6d055d16c7620b7804b6a46cb481d00b3dbb5007Eugen Kuksa

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksavoid maildir_uidlist_deinit(struct maildir_uidlist **_uidlist)

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa{

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa struct maildir_uidlist *uidlist = *_uidlist;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa i_assert(!UIDLIST_IS_LOCKED(uidlist));

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa *_uidlist = NULL;

26e23083feb1577ee4bd102bc09ca440d20548b9Eugen Kuksa maildir_uidlist_update(uidlist);

26e23083feb1577ee4bd102bc09ca440d20548b9Eugen Kuksa maildir_uidlist_close(uidlist);

26e23083feb1577ee4bd102bc09ca440d20548b9Eugen Kuksa

26e23083feb1577ee4bd102bc09ca440d20548b9Eugen Kuksa hash_destroy(&uidlist->files);

26e23083feb1577ee4bd102bc09ca440d20548b9Eugen Kuksa if (uidlist->record_pool != NULL)

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa pool_unref(&uidlist->record_pool);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa array_free(&uidlist->records);

2da9df454152009d7a5e0d83d519aa2bc9844233Eugen Kuksa str_free(&uidlist->hdr_extensions);

2da9df454152009d7a5e0d83d519aa2bc9844233Eugen Kuksa i_free(uidlist->path);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa i_free(uidlist);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksastatic int maildir_uid_cmp(const void *p1, const void *p2)

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa{

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa const struct maildir_uidlist_rec *const *rec1 = p1, *const *rec2 = p2;

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa return (*rec1)->uid < (*rec2)->uid ? -1 :

0978722f6ca27f5d5f5ed4ec8400703dfe211184Eugen Kuksa (*rec1)->uid > (*rec2)->uid ? 1 : 0;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa}

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

e6edf613be834fd6ab5ea816077d4263609db07eEugen Kuksastatic void

0b99be5ced371cca00283694e1bd53a8ac0d7b5dEugen Kuksamaildir_uidlist_records_array_delete(struct maildir_uidlist *uidlist,

1724834ae596f84e8237a3bba5a8af15916a8ee9Eugen Kuksa struct maildir_uidlist_rec *rec)

1724834ae596f84e8237a3bba5a8af15916a8ee9Eugen Kuksa{

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa struct maildir_uidlist_rec *const *recs, *const *pos;

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa unsigned int count;

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa recs = array_get(&uidlist->records, &count);

1724834ae596f84e8237a3bba5a8af15916a8ee9Eugen Kuksa pos = bsearch(&rec, recs, count, sizeof(*recs), maildir_uid_cmp);

1724834ae596f84e8237a3bba5a8af15916a8ee9Eugen Kuksa i_assert(pos != NULL);

1724834ae596f84e8237a3bba5a8af15916a8ee9Eugen Kuksa

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa array_delete(&uidlist->records, pos - recs, 1);

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksa}

1724834ae596f84e8237a3bba5a8af15916a8ee9Eugen Kuksa

df4ff7ec6be98e7ae5830731becc1a3d55105378Eugen Kuksastatic bool

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksamaildir_uidlist_read_extended(struct maildir_uidlist *uidlist,

b1fe9054ad7c7192fe4c474363247dad15963e99Eugen Kuksa const char **line_p,

57c81a32dff6182f040b4f852892144641a62fe5Eugen Kuksa struct maildir_uidlist_rec *rec)

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa{

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa const char *start, *line = *line_p;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa buffer_t *buf;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa buf = buffer_create_dynamic(pool_datastack_create(), 128);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa while (*line != '\0' && *line != ':') {

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa /* skip over an extension field */

4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa start = line;

4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa while (*line != ' ' && *line != '\0') line++;

4634cde5d3428bd5ab34b8212ac2f4637cdfff6fEugen Kuksa buffer_append(buf, start, line - start);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa buffer_append_c(buf, '\0');

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa while (*line == ' ') line++;

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa if (buf->used > 0) {

5efadb4662f2a63d5f5f1a5b303ab7c3371069a8Eugen Kuksa /* save the extensions */

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa buffer_append_c(buf, '\0');

923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa rec->extensions = p_malloc(uidlist->record_pool, buf->used);

923d69139038e74c0936e826bbfdc8717fbbc7b3Eugen Kuksa memcpy(rec->extensions, buf->data, buf->used);

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa }

daf3e28fff47a65b53d6fb65155301763b9f166eEugen Kuksa

if (*line == ':')

line++;

if (*line == '\0')

return FALSE;

*line_p = line;

return TRUE;

}

static void ATTR_FORMAT(2, 3)

maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist,

const char *fmt, ...)

{

struct mail_storage *storage = uidlist->ibox->box.storage;

va_list args;

va_start(args, fmt);

mail_storage_set_critical(storage, "Broken file %s line %u: %s",

uidlist->path, uidlist->read_line_count,

t_strdup_vprintf(fmt, args));

va_end(args);

}

static int maildir_uidlist_next(struct maildir_uidlist *uidlist,

const char *line)

{

struct maildir_uidlist_rec *rec, *old_rec;

uint32_t uid;

uid = 0;

while (*line >= '0' && *line <= '9') {

uid = uid*10 + (*line - '0');

line++;

}

if (uid == 0 || *line != ' ') {

/* invalid file */

maildir_uidlist_set_corrupted(uidlist, "Invalid data: %s",

line);

return 0;

}

if (uid <= uidlist->prev_read_uid) {

maildir_uidlist_set_corrupted(uidlist,

"UIDs not ordered (%u > %u)",

uid, uidlist->prev_read_uid);

return 0;

}

uidlist->prev_read_uid = uid;

if (uid <= uidlist->last_seen_uid) {

/* we already have this */

return 1;

}

uidlist->last_seen_uid = uid;

if (uid >= uidlist->next_uid && uidlist->version == 1) {

maildir_uidlist_set_corrupted(uidlist,

"UID larger than next_uid (%u >= %u)",

uid, uidlist->next_uid);

return 0;

}

rec = p_new(uidlist->record_pool, struct maildir_uidlist_rec, 1);

rec->uid = uid;

rec->flags = MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;

while (*line == ' ') line++;

if (uidlist->version == 3) {

/* read extended fields */

bool ret;

T_BEGIN {

ret = maildir_uidlist_read_extended(uidlist, &line,

rec);

} T_END;

if (!ret) {

maildir_uidlist_set_corrupted(uidlist,

"Invalid extended fields: %s", line);

return 0;

}

}

old_rec = hash_lookup(uidlist->files, line);

if (old_rec != NULL) {

/* This can happen if expunged file is moved back and the file

i_warning("%s: Duplicate file entry at line %u: %s",

uidlist->path, uidlist->read_line_count, line);

/* Delete the old UID */

maildir_uidlist_records_array_delete(uidlist, old_rec);

/* Replace the old record with this new one */

*old_rec = *rec;

rec = old_rec;

uidlist->recreate = TRUE;

}

rec->filename = p_strdup(uidlist->record_pool, line);

hash_insert(uidlist->files, rec->filename, rec);

array_append(&uidlist->records, &rec, 1);

return 1;

}

static int maildir_uidlist_read_header(struct maildir_uidlist *uidlist,

struct istream *input)

{

unsigned int uid_validity, next_uid;

string_t *ext_hdr;

const char *line;

char key;

line = i_stream_read_next_line(input);

if (line == NULL) {

/* I/O error / empty file */

return input->stream_errno == 0 ? 0 : -1;

}

uidlist->read_line_count = 1;

if (*line < '0' || *line > '9' || line[1] != ' ') {

maildir_uidlist_set_corrupted(uidlist,

"Corrupted header (invalid version number)");

return 0;

}

uidlist->version = *line - '0';

line += 2;

switch (uidlist->version) {

case 1:

if (sscanf(line, "%u %u", &uid_validity, &next_uid) != 2) {

maildir_uidlist_set_corrupted(uidlist,

"Corrupted header (version 1)");

return 0;

}

if (uid_validity == uidlist->uid_validity &&

next_uid < uidlist->next_uid) {

maildir_uidlist_set_corrupted(uidlist,

"next_uid was lowered (v1, %u -> %u)",

uidlist->next_uid, next_uid);

return 0;

}

break;

case 3:

ext_hdr = uidlist->hdr_extensions;

str_truncate(ext_hdr, 0);

while (*line != '\0') T_BEGIN {

const char *value;

key = *line;

value = ++line;

while (*line != '\0' && *line != ' ') line++;

value = t_strdup_until(value, line);

switch (key) {

case MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY:

uid_validity = strtoul(value, NULL, 10);

break;

case MAILDIR_UIDLIST_HDR_EXT_NEXT_UID:

next_uid = strtoul(value, NULL, 10);

break;

default:

if (str_len(ext_hdr) > 0)

str_append_c(ext_hdr, ' ');

str_printfa(ext_hdr, "%c%s", key, value);

break;

}

while (*line == ' ') line++;

} T_END;

break;

default:

maildir_uidlist_set_corrupted(uidlist, "Unsupported version %u",

uidlist->version);

return 0;

}

if (uid_validity == 0 || next_uid == 0) {

maildir_uidlist_set_corrupted(uidlist,

"Broken header (uidvalidity = %u, next_uid=%u)",

uid_validity, next_uid);

return 0;

}

uidlist->uid_validity = uid_validity;

uidlist->next_uid = next_uid;

return 1;

}

static int

maildir_uidlist_update_read(struct maildir_uidlist *uidlist,

bool *retry_r, bool try_retry)

{

struct mail_storage *storage = uidlist->ibox->box.storage;

const char *line;

unsigned int orig_next_uid;

struct istream *input;

struct stat st;

uoff_t last_read_offset;

int fd, ret;

*retry_r = FALSE;

if (uidlist->fd == -1) {

fd = nfs_safe_open(uidlist->path, O_RDWR);

if (fd == -1) {

if (errno != ENOENT) {

mail_storage_set_critical(storage,

"open(%s) failed: %m", uidlist->path);

return -1;

}

return 0;

}

last_read_offset = 0;

} else {

/* the file was updated */

fd = uidlist->fd;

if (lseek(fd, 0, SEEK_SET) < 0) {

mail_storage_set_critical(storage,

"lseek(%s) failed: %m", uidlist->path);

return -1;

}

uidlist->fd = -1;

uidlist->fd_ino = 0;

last_read_offset = uidlist->last_read_offset;

uidlist->last_read_offset = 0;

}

if (fstat(fd, &st) < 0) {

close_keep_errno(fd);

if (errno == ESTALE && try_retry) {

*retry_r = TRUE;

return -1;

}

mail_storage_set_critical(storage,

"fstat(%s) failed: %m", uidlist->path);

return -1;

}

if (uidlist->record_pool == NULL) {

uidlist->record_pool =

pool_alloconly_create(MEMPOOL_GROWING

"uidlist record_pool",

nearest_power(st.st_size -

st.st_size/8));

}

input = i_stream_create_fd(fd, 4096, FALSE);

i_stream_seek(input, last_read_offset);

orig_next_uid = uidlist->next_uid;

ret = input->v_offset != 0 ? 1 :

maildir_uidlist_read_header(uidlist, input);

if (ret > 0) {

uidlist->prev_read_uid = 0;

uidlist->change_counter++;

uidlist->read_records_count = 0;

ret = 1;

while ((line = i_stream_read_next_line(input)) != NULL) {

uidlist->read_records_count++;

uidlist->read_line_count++;

if (!maildir_uidlist_next(uidlist, line)) {

ret = 0;

break;

}

}

if (input->stream_errno != 0)

ret = -1;

if (uidlist->next_uid <= uidlist->prev_read_uid)

uidlist->next_uid = uidlist->prev_read_uid + 1;

if (uidlist->next_uid < orig_next_uid) {

mail_storage_set_critical(storage,

"%s: next_uid was lowered (%u -> %u)",

uidlist->path, orig_next_uid,

uidlist->next_uid);

uidlist->recreate = TRUE;

uidlist->next_uid = orig_next_uid;

}

}

if (ret == 0) {

/* file is broken */

(void)unlink(uidlist->path);

} else if (ret > 0) {

/* success */

uidlist->fd = fd;

uidlist->fd_dev = st.st_dev;

uidlist->fd_ino = st.st_ino;

uidlist->fd_size = st.st_size;

uidlist->last_read_offset = input->v_offset;

} else {

/* I/O error */

if (input->stream_errno == ESTALE && try_retry)

*retry_r = TRUE;

else {

errno = input->stream_errno;

mail_storage_set_critical(storage,

"read(%s) failed: %m", uidlist->path);

}

}

i_stream_destroy(&input);

if (ret <= 0) {

if (close(fd) < 0)

i_error("close(%s) failed: %m", uidlist->path);

}

return ret;

}

static int

maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r)

{

struct mail_storage *storage = uidlist->ibox->box.storage;

struct stat st;

*recreated_r = FALSE;

if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_STORAGE) != 0) {

nfs_flush_file_handle_cache(uidlist->path);

nfs_flush_attr_cache_unlocked(uidlist->path);

}

if (nfs_safe_stat(uidlist->path, &st) < 0) {

if (errno != ENOENT) {

mail_storage_set_critical(storage,

"stat(%s) failed: %m", uidlist->path);

return -1;

}

return 0;

}

if (st.st_ino != uidlist->fd_ino ||

!CMP_DEV_T(st.st_dev, uidlist->fd_dev)) {

/* file recreated */

*recreated_r = TRUE;

return 1;

}

if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_STORAGE) != 0) {

/* NFS: either the file hasn't been changed, or it has already

if (fstat(uidlist->fd, &st) < 0) {

if (errno == ESTALE) {

*recreated_r = TRUE;

return 1;

}

mail_storage_set_critical(storage,

"fstat(%s) failed: %m", uidlist->path);

return -1;

}

}

if (st.st_size != uidlist->fd_size) {

/* file modified but not recreated */

return 1;

} else {

/* unchanged */

return 0;

}

}

int maildir_uidlist_refresh(struct maildir_uidlist *uidlist)

{

unsigned int i;

bool retry, recreated;

int ret;

if (uidlist->fd != -1) {

ret = maildir_uidlist_has_changed(uidlist, &recreated);

if (ret <= 0)

return ret;

if (recreated)

maildir_uidlist_close(uidlist);

}

for (i = 0; ; i++) {

ret = maildir_uidlist_update_read(uidlist, &retry,

i < UIDLIST_ESTALE_RETRY_COUNT);

if (!retry)

break;

/* ESTALE - try reopening and rereading */

maildir_uidlist_close(uidlist);

}

if (ret >= 0)

uidlist->initial_read = TRUE;

return ret;

}

static struct maildir_uidlist_rec *

maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid,

unsigned int *idx_r)

{

struct maildir_uidlist_rec *const *recs;

unsigned int idx, left_idx, right_idx;

if (!uidlist->initial_read) {

/* first time we need to read uidlist */

if (maildir_uidlist_refresh(uidlist) < 0)

return NULL;

}

idx = left_idx = 0;

recs = array_get(&uidlist->records, &right_idx);

while (left_idx < right_idx) {

idx = (left_idx + right_idx) / 2;

if (recs[idx]->uid < uid)

left_idx = idx+1;

else if (recs[idx]->uid > uid)

right_idx = idx;

else {

*idx_r = idx;

return recs[idx];

}

}

if (idx > 0) idx--;

*idx_r = idx;

return NULL;

}

const char *

maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,

enum maildir_uidlist_rec_flag *flags_r)

{

const char *fname;

fname = maildir_uidlist_lookup_nosync(uidlist, uid, flags_r);

if (fname == NULL) {

if (uidlist->fd != -1 || uidlist->mbox == NULL)

return NULL;

/* the uidlist doesn't exist. */

if (maildir_storage_sync_force(uidlist->mbox, uid) < 0)

return NULL;

/* try again */

fname = maildir_uidlist_lookup_nosync(uidlist, uid, flags_r);

}

return fname;

}

const char *

maildir_uidlist_lookup_nosync(struct maildir_uidlist *uidlist, uint32_t uid,

enum maildir_uidlist_rec_flag *flags_r)

{

const struct maildir_uidlist_rec *rec;

unsigned int idx;

rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);

if (rec == NULL)

return NULL;

*flags_r = rec->flags;

return rec->filename;

}

const char *

maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid,

enum maildir_uidlist_rec_ext_key key)

{

const struct maildir_uidlist_rec *rec;

unsigned int idx;

const unsigned char *p;

const char *value;

rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);

if (rec == NULL || rec->extensions == NULL)

return NULL;

p = rec->extensions; value = NULL;

while (*p != '\0') {

/* <key><value>\0 */

if (*p == (char)key)

return (const char *)p + 1;

p += strlen((const char *)p) + 1;

}

return NULL;

}

uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist)

{

return uidlist->uid_validity;

}

uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist)

{

return !uidlist->initial_read ? 0 : uidlist->next_uid;

}

void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist,

uint32_t uid_validity)

{

i_assert(uid_validity != 0);

uidlist->uid_validity = uid_validity;

}

void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist,

uint32_t next_uid, bool force)

{

if (uidlist->next_uid < next_uid || force)

uidlist->next_uid = next_uid;

}

static void

maildir_uidlist_set_ext_real(struct maildir_uidlist *uidlist, uint32_t uid,

enum maildir_uidlist_rec_ext_key key,

const char *value)

{

struct maildir_uidlist_rec *rec;

unsigned int idx;

const unsigned char *p;

buffer_t *buf;

unsigned int len;

rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);

if (rec == NULL) {

/* maybe it's a new message */

if (maildir_uidlist_refresh(uidlist) < 0)

return;

rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);

if (rec == NULL) {

/* message is already expunged, ignore */

return;

}

}

buf = buffer_create_dynamic(pool_datastack_create(), 128);

/* copy existing extensions, except for the one we're updating */

if (rec->extensions != NULL) {

p = rec->extensions;

while (*p != '\0') {

/* <key><value>\0 */

len = strlen((const char *)p) + 1;

if (*p != (char)key)

buffer_append(buf, p, len);

p += len;

}

}

buffer_append_c(buf, key);

buffer_append(buf, value, strlen(value) + 1);

buffer_append_c(buf, '\0');

rec->extensions = p_malloc(uidlist->record_pool, buf->used);

memcpy(rec->extensions, buf->data, buf->used);

uidlist->recreate = TRUE;

}

void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid,

enum maildir_uidlist_rec_ext_key key,

const char *value)

{

T_BEGIN {

maildir_uidlist_set_ext_real(uidlist, uid, key, value);

} T_END;

}

static int maildir_uidlist_write_fd(struct maildir_uidlist *uidlist, int fd,

const char *path, unsigned int first_idx,

uoff_t *file_size_r)

{

struct mail_storage *storage = uidlist->ibox->box.storage;

struct maildir_uidlist_iter_ctx *iter;

struct ostream *output;

struct maildir_uidlist_rec *rec;

string_t *str;

const unsigned char *p;

unsigned int len;

int ret;

i_assert(fd != -1);

output = o_stream_create_fd_file(fd, (uoff_t)-1, FALSE);

o_stream_cork(output);

str = t_str_new(512);

if (output->offset == 0) {

i_assert(first_idx == 0);

uidlist->version = 3;

i_assert(uidlist->uid_validity != 0);

i_assert(uidlist->next_uid > 0);

str_printfa(str, "%u V%u N%u", uidlist->version,

uidlist->uid_validity, uidlist->next_uid);

if (str_len(uidlist->hdr_extensions) > 0) {

str_append_c(str, ' ');

str_append_str(str, uidlist->hdr_extensions);

}

str_append_c(str, '\n');

o_stream_send(output, str_data(str), str_len(str));

}

iter = maildir_uidlist_iter_init(uidlist);

i_assert(first_idx <= array_count(&uidlist->records));

iter->next += first_idx;

while (maildir_uidlist_iter_next_rec(iter, &rec)) {

str_truncate(str, 0);

str_printfa(str, "%u", rec->uid);

if (rec->extensions != NULL) {

for (p = rec->extensions; *p != '\0'; ) {

len = strlen((const char *)p);

str_append_c(str, ' ');

str_append_n(str, p, len);

p += len + 1;

}

}

str_printfa(str, " :%s\n", rec->filename);

o_stream_send(output, str_data(str), str_len(str));

}

maildir_uidlist_iter_deinit(&iter);

o_stream_flush(output);

ret = output->stream_errno == 0 ? 0 : -1;

*file_size_r = output->offset;

o_stream_unref(&output);

if (ret < 0) {

mail_storage_set_critical(storage,

"o_stream_send(%s) failed: %m", path);

(void)close(fd);

return -1;

}

if (!uidlist->ibox->fsync_disable) {

if (fdatasync(fd) < 0) {

mail_storage_set_critical(storage,

"fdatasync(%s) failed: %m", path);

(void)close(fd);

return -1;

}

}

return 0;

}

static int maildir_uidlist_recreate(struct maildir_uidlist *uidlist)

{

struct mailbox *box = &uidlist->ibox->box;

const char *control_dir, *temp_path;

struct stat st;

mode_t old_mask;

uoff_t file_size;

int i, fd, ret;

control_dir = mailbox_list_get_path(box->storage->list, box->name,

MAILBOX_LIST_PATH_TYPE_CONTROL);

temp_path = t_strconcat(control_dir,

"/" MAILDIR_UIDLIST_NAME ".tmp", NULL);

for (i = 0;; i++) {

old_mask = umask(0777 & ~box->file_create_mode);

fd = open(temp_path, O_RDWR | O_CREAT | O_TRUNC, 0777);

umask(old_mask);

if (fd != -1)

break;

if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT ||

uidlist->mbox == NULL) {

mail_storage_set_critical(box->storage,

"open(%s, O_CREAT) failed: %m", temp_path);

return -1;

}

/* the control dir doesn't exist. create it unless the whole

if (maildir_set_deleted(uidlist->mbox))

return -1;

}

if (box->file_create_gid != (gid_t)-1) {

if (fchown(fd, (uid_t)-1, box->file_create_gid) < 0) {

mail_storage_set_critical(box->storage,

"fchown(%s) failed: %m", temp_path);

}

}

ret = maildir_uidlist_write_fd(uidlist, fd, temp_path, 0, &file_size);

if (ret == 0) {

if (rename(temp_path, uidlist->path) < 0) {

mail_storage_set_critical(box->storage,

"rename(%s, %s) failed: %m",

temp_path, uidlist->path);

ret = -1;

}

}

if (ret < 0) {

if (unlink(temp_path) < 0) {

mail_storage_set_critical(box->storage,

"unlink(%s) failed: %m", temp_path);

}

} else if (fstat(fd, &st) < 0) {

i_error("fstat(%s) failed: %m", temp_path);

(void)close(fd);

ret = -1;

} else if (file_size != (uoff_t)st.st_size) {

i_assert(!file_dotlock_is_locked(uidlist->dotlock));

i_error("Maildir uidlist dotlock overridden: %s",

uidlist->path);

(void)close(fd);

ret = -1;

} else {

maildir_uidlist_close(uidlist);

uidlist->fd = fd;

uidlist->fd_dev = st.st_dev;

uidlist->fd_ino = st.st_ino;

uidlist->fd_size = st.st_size;

uidlist->last_read_offset = st.st_size;

uidlist->recreate = FALSE;

}

return ret;

}

int maildir_uidlist_update(struct maildir_uidlist *uidlist)

{

int ret;

if (!uidlist->recreate)

return 0;

if (maildir_uidlist_lock(uidlist) <= 0)

return -1;

ret = maildir_uidlist_recreate(uidlist);

maildir_uidlist_unlock(uidlist);

return ret;

}

static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx)

{

struct maildir_uidlist *uidlist = ctx->uidlist;

uoff_t file_size;

if (uidlist->uid_validity == 0) {

/* saving a message to a newly created maildir. */

const struct mail_index_header *hdr;

hdr = mail_index_get_header(uidlist->ibox->view);

uidlist->uid_validity = hdr->uid_validity != 0 ?

hdr->uid_validity : (uint32_t)ioloop_time;

}

if (ctx->uidlist->recreate || uidlist->fd == -1 ||

uidlist->version != 3 ||

ctx->finish_change_counter != ctx->uidlist->change_counter ||

(uidlist->read_records_count + ctx->new_files_count) *

UIDLIST_COMPRESS_PERCENTAGE / 100 >= array_count(&uidlist->records))

return maildir_uidlist_recreate(uidlist);

i_assert(ctx->first_unwritten_pos != (unsigned int)-1);

if (lseek(uidlist->fd, 0, SEEK_END) < 0) {

mail_storage_set_critical(uidlist->ibox->box.storage,

"lseek(%s) failed: %m", uidlist->path);

return -1;

}

if (maildir_uidlist_write_fd(uidlist, uidlist->fd, uidlist->path,

ctx->first_unwritten_pos, &file_size) < 0)

return -1;

uidlist->last_read_offset = file_size;

return 0;

}

static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist,

bool nonsynced)

{

struct maildir_uidlist_rec **recs;

unsigned int i, count;

recs = array_get_modifiable(&uidlist->records, &count);

if (nonsynced) {

for (i = 0; i < count; i++)

recs[i]->flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;

} else {

for (i = 0; i < count; i++)

recs[i]->flags &= ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;

}

}

int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,

enum maildir_uidlist_sync_flags sync_flags,

struct maildir_uidlist_sync_ctx **sync_ctx_r)

{

struct maildir_uidlist_sync_ctx *ctx;

bool locked;

int ret;

if ((sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) == 0) {

if ((ret = maildir_uidlist_lock(uidlist)) <= 0)

return ret;

} else {

if ((ret = maildir_uidlist_try_lock(uidlist)) < 0)

return -1;

if (ret == 0) {

/* couldn't lock it */

if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0)

return 0;

/* forcing the lock */

}

}

locked = ret > 0;

if (!locked) {

if (maildir_uidlist_refresh(uidlist) < 0)

return -1;

}

*sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1);

ctx->uidlist = uidlist;

ctx->sync_flags = sync_flags;

ctx->partial = !locked ||

(sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0;

ctx->locked = locked;

ctx->first_unwritten_pos = (unsigned int)-1;

ctx->first_nouid_pos = (unsigned int)-1;

if (ctx->partial) {

if ((sync_flags & MAILDIR_UIDLIST_SYNC_KEEP_STATE) == 0) {

/* initially mark all nonsynced */

maildir_uidlist_mark_all(uidlist, TRUE);

}

return 1;

}

ctx->record_pool = pool_alloconly_create(MEMPOOL_GROWING

"maildir_uidlist_sync", 16384);

ctx->files = hash_create(default_pool, ctx->record_pool, 4096,

maildir_filename_base_hash,

maildir_filename_base_cmp);

i_array_init(&ctx->records, array_count(&uidlist->records));

return 1;

}

static void

maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,

const char *filename,

enum maildir_uidlist_rec_flag flags)

{

struct maildir_uidlist *uidlist = ctx->uidlist;

struct maildir_uidlist_rec *rec;

/* we'll update uidlist directly */

rec = hash_lookup(uidlist->files, filename);

if (rec == NULL) {

/* doesn't exist in uidlist */

if (!ctx->locked) {

/* we can't add it, so just ignore it */

return;

}

if (ctx->first_nouid_pos == (unsigned int)-1)

ctx->first_nouid_pos = array_count(&uidlist->records);

ctx->new_files_count++;

ctx->changed = TRUE;

if (uidlist->record_pool == NULL) {

uidlist->record_pool =

pool_alloconly_create(MEMPOOL_GROWING

"uidlist record_pool",

1024);

}

rec = p_new(uidlist->record_pool,

struct maildir_uidlist_rec, 1);

rec->uid = (uint32_t)-1;

array_append(&uidlist->records, &rec, 1);

uidlist->change_counter++;

}

rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;

rec->filename = p_strdup(uidlist->record_pool, filename);

hash_insert(uidlist->files, rec->filename, rec);

ctx->finished = FALSE;

}

static unsigned char *ext_dup(pool_t pool, const unsigned char *extensions)

{

unsigned char *ret;

if (extensions == NULL)

return NULL;

T_BEGIN {

unsigned int len;

for (len = 0; extensions[len] != '\0'; len++) {

while (extensions[len] != '\0') len++;

}

ret = p_malloc(pool, len + 1);

memcpy(ret, extensions, len);

} T_END;

return ret;

}

int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,

const char *filename,

enum maildir_uidlist_rec_flag flags)

{

struct maildir_uidlist *uidlist = ctx->uidlist;

struct maildir_uidlist_rec *rec, *old_rec;

if (ctx->failed)

return -1;

if (ctx->partial) {

maildir_uidlist_sync_next_partial(ctx, filename, flags);

return 1;

}

rec = hash_lookup(ctx->files, filename);

if (rec != NULL) {

if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |

MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) {

/* possibly duplicate */

return 0;

}

/* probably was in new/ and now we're seeing it in cur/.

rec->flags &= ~(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |

MAILDIR_UIDLIST_REC_FLAG_MOVED);

} else {

old_rec = hash_lookup(uidlist->files, filename);

i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist));

rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1);

if (old_rec != NULL) {

*rec = *old_rec;

rec->extensions =

ext_dup(ctx->record_pool, rec->extensions);

} else {

rec->uid = (uint32_t)-1;

ctx->new_files_count++;

ctx->changed = TRUE;

/* didn't exist in uidlist, it's recent */

flags |= MAILDIR_UIDLIST_REC_FLAG_RECENT;

}

array_append(&ctx->records, &rec, 1);

}

rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;