maildir-uidlist.c revision aba2f05e938a7f635b3f114d56c7c3413ee858fe
/* Copyright (c) 2003-2010 Dovecot authors, see the included COPYING file */
/*
Version 1 format has been used for most versions of Dovecot up to v1.0.x.
It's also compatible with Courier IMAP's courierimapuiddb file.
The format is:
header: 1 <uid validity> <next uid>
entry: <uid> <filename>
--
Version 2 format was written by a few development Dovecot versions, but
v1.0.x still parses the format. The format has <flags> field after <uid>.
--
Version 3 format is an extensible format used by Dovecot v1.1 and later.
It's also parsed by v1.0.2 (and later). The format is:
header: 3 [<key><value> ...]
entry: <uid> [<key><value> ...] :<filename>
See enum maildir_uidlist_*_ext_key for used keys.
*/
#include "lib.h"
#include "array.h"
#include "hash.h"
#include "istream.h"
#include "ostream.h"
#include "str.h"
#include "hex-binary.h"
#include "file-dotlock.h"
#include "close-keep-errno.h"
#include "nfs-workarounds.h"
#include "eacces-error.h"
#include "maildir-storage.h"
#include "maildir-sync.h"
#include "maildir-filename.h"
#include "maildir-uidlist.h"
#include <stdio.h>
#include <stdlib.h>
/* NFS: How many times to retry reading dovecot-uidlist file if ESTALE
error occurs in the middle of reading it */
#define UIDLIST_VERSION 3
#define UIDLIST_COMPRESS_PERCENTAGE 75
#define UIDLIST_IS_LOCKED(uidlist) \
((uidlist)->lock_count > 0)
#define UIDLIST_ALLOW_WRITING(uidlist) \
struct maildir_uidlist_rec {
char *filename;
unsigned char *extensions; /* <data>\0[<data>\0 ...]\0 */
};
struct maildir_uidlist {
struct maildir_mailbox *mbox;
struct index_mailbox *ibox;
char *path;
int fd;
unsigned int lock_count;
struct dotlock_settings dotlock_settings;
struct hash_table *files;
unsigned int change_counter;
unsigned int version;
unsigned int hdr_next_uid;
unsigned int read_records_count, read_line_count;
unsigned int recreate:1;
unsigned int initial_read:1;
unsigned int initial_hdr_read:1;
unsigned int retry_rewind:1;
unsigned int locked_refresh:1;
unsigned int unsorted:1;
unsigned int have_mailbox_guid:1;
};
struct maildir_uidlist_sync_ctx {
struct maildir_uidlist *uidlist;
struct hash_table *files;
unsigned int first_unwritten_pos, first_new_pos;
unsigned int new_files_count;
unsigned int finish_change_counter;
unsigned int partial:1;
unsigned int finished:1;
unsigned int changed:1;
unsigned int failed:1;
unsigned int locked:1;
};
struct maildir_uidlist_iter_ctx {
struct maildir_uidlist *uidlist;
unsigned int change_counter;
};
struct maildir_uidlist_rec **rec_r);
bool refresh_when_locked)
{
const char *control_dir, *path;
const enum dotlock_create_flags dotlock_flags =
int i, ret;
if (uidlist->lock_count > 0) {
if (maildir_uidlist_refresh(uidlist) < 0)
return -1;
}
uidlist->lock_count++;
return 1;
}
for (i = 0;; i++) {
if (ret > 0)
break;
/* failure */
if (ret == 0) {
return 0;
}
} else {
"file_dotlock_create(%s) failed: %m",
path);
}
return -1;
}
/* the control dir doesn't exist. create it unless the whole
mailbox was just deleted. */
return -1;
}
uidlist->lock_count++;
if (refresh) {
/* make sure we have the latest changes before
changing anything */
if (maildir_uidlist_refresh(uidlist) < 0) {
return -1;
}
}
return 1;
}
{
}
{
}
{
}
{
return UIDLIST_IS_LOCKED(uidlist);
}
{
if (--uidlist->lock_count > 0)
return;
}
{
return TRUE;
}
struct maildir_uidlist *
{
struct maildir_uidlist *uidlist;
const char *control_dir;
return uidlist;
}
{
struct maildir_uidlist *uidlist;
return uidlist;
}
{
}
}
uidlist->last_read_offset = 0;
uidlist->read_line_count = 0;
}
{
uidlist->last_seen_uid = 0;
}
{
}
struct maildir_uidlist_rec *const *rec2)
{
}
const char *fmt, ...)
{
if (uidlist->retry_rewind) {
"Broken or unexpectedly changed file %s "
"line %u: %s - re-reading from beginning",
} else {
}
}
{
struct maildir_index_header *mhdr;
/* dbox is using this */
return;
}
/* upgrading from older verson. don't update the
uidlist times until it uses the new format */
return;
}
}
static unsigned int
struct maildir_uidlist_rec *rec)
{
} else {
break;
}
}
return idx;
}
static bool
const char **line_p,
struct maildir_uidlist_rec *rec)
{
/* skip over an extension field */
}
/* save the extensions */
}
if (*line == ':')
line++;
if (*line == '\0')
return FALSE;
return TRUE;
}
const char *line)
{
unsigned int count;
uid = 0;
line++;
}
/* invalid file */
line);
return FALSE;
}
"UIDs not ordered (%u >= %u)",
return FALSE;
}
"UID too high (%u)", uid);
return FALSE;
}
/* we already have this */
return TRUE;
}
"UID larger than next_uid (%u >= %u)",
return FALSE;
}
/* read extended fields */
bool ret;
T_BEGIN {
rec);
} T_END;
if (!ret) {
"Invalid extended fields: %s", line);
return FALSE;
}
}
"%s: Broken filename at line %u: %s",
return FALSE;
}
/* no conflicts */
/* most likely this is a record we saved ourself, but couldn't
update last_seen_uid because uidlist wasn't refreshed while
it was locked.
another possibility is a duplicate file record. currently
it would be a bug, but not that big of a deal. also perhaps
in future such duplicate lines could be used to update
extended fields. so just let it through anyway.
we'll waste a bit of memory here by allocating the record
twice, but that's not really a problem. */
return TRUE;
} else {
/* This can happen if expunged file is moved back and the file
was appended to uidlist. */
i_warning("%s: Duplicate file entry at line %u: "
"%s (uid %u -> %u)",
/* Delete the old UID */
/* Replace the old record with this new one */
}
/* we most likely have some records in the array that we saved
ourself without refreshing uidlist */
}
return TRUE;
}
static int
const char *line,
unsigned int *uid_validity_r,
unsigned int *next_uid_r)
{
char key;
while (*line != '\0') {
const char *value;
switch (key) {
break;
break;
"Invalid mailbox GUID: %s", value);
return -1;
}
sizeof(uidlist->mailbox_guid));
break;
default:
break;
}
}
return 0;
}
{
unsigned int uid_validity = 0, next_uid = 0;
const char *line;
int ret;
/* I/O error / empty file */
}
"Corrupted header (invalid version number)");
return 0;
}
line += 2;
case 1:
"Corrupted header (version 1)");
return 0;
}
break;
case UIDLIST_VERSION:
T_BEGIN {
&next_uid);
} T_END;
if (ret < 0)
return 0;
break;
default:
return 0;
}
if (uid_validity == 0 || next_uid == 0) {
"Broken header (uidvalidity = %u, next_uid=%u)",
return 0;
}
"next_uid header was lowered (%u -> %u)",
return 0;
}
return 1;
}
{
}
static int
{
const char *line;
if (fd == -1) {
return -1;
}
return 0;
}
last_read_offset = 0;
} else {
/* the file was updated */
return -1;
}
return -1;
}
uidlist->last_read_offset = 0;
}
return -1;
}
return -1;
}
"uidlist record_pool",
}
if (ret > 0) {
uidlist->prev_read_uid = 0;
uidlist->read_records_count = 0;
ret = 1;
if (!uidlist->retry_rewind)
ret = 0;
else {
ret = -1;
}
break;
}
}
if (input->stream_errno != 0)
ret = -1;
}
orig_uid_validity != 0) {
"%s: next_uid was lowered (%u -> %u, hdr=%u)",
}
}
if (ret == 0) {
/* file is broken */
} else if (ret > 0) {
/* success */
} else if (!*retry_r) {
/* I/O error */
else {
}
uidlist->last_read_offset = 0;
}
if (ret <= 0) {
}
}
return ret;
}
static int
{
}
return -1;
}
return 0;
}
return 1;
}
static int
{
int ret;
*recreated_r = FALSE;
/* file recreated */
*recreated_r = TRUE;
return 1;
}
/* NFS: either the file hasn't been changed, or it has already
been deleted and the inodes just happen to be the same.
check if the fd is still valid. */
*recreated_r = TRUE;
return 1;
}
return -1;
}
}
/* file modified but not recreated */
return 1;
} else {
/* unchanged */
return 0;
}
}
{
bool recreated;
int ret;
if (ret <= 0) {
if (UIDLIST_IS_LOCKED(uidlist))
}
if (!recreated)
return 0;
}
return -1;
}
return 0;
}
{
unsigned int i;
bool retry;
int ret;
if (maildir_uidlist_open_latest(uidlist) < 0)
return -1;
for (i = 0; ; i++) {
if (!retry)
break;
/* ESTALE - try reopening and rereading */
}
if (ret >= 0) {
if (UIDLIST_IS_LOCKED(uidlist))
if (!uidlist->have_mailbox_guid) {
(void)maildir_uidlist_update(uidlist);
}
}
return ret;
}
{
struct mail_index_view *view;
const struct mail_index_header *hdr;
int ret;
return maildir_uidlist_refresh(uidlist);
return ret;
/* index is up-to-date. look up the uidvalidity and next-uid
from it. we'll need to create a new view temporarily to
make sure we get the latest values. */
if (UIDLIST_IS_LOCKED(uidlist))
return 1;
} else {
return maildir_uidlist_refresh(uidlist);
}
}
static int
unsigned int *idx_r,
struct maildir_uidlist_rec **rec_r)
{
struct maildir_uidlist_rec *const *recs;
if (!uidlist->initial_read) {
/* first time we need to read uidlist */
if (maildir_uidlist_refresh(uidlist) < 0)
return -1;
}
else {
return 1;
}
}
return 0;
}
enum maildir_uidlist_rec_flag *flags_r,
const char **fname_r)
{
int ret;
if (ret <= 0) {
if (ret < 0)
return -1;
/* refresh uidlist and check again in case it was added
after the last mailbox sync */
if (maildir_uidlist_refresh(uidlist) < 0)
return -1;
} else {
/* the uidlist doesn't exist. */
return -1;
}
/* try again */
}
return ret;
}
enum maildir_uidlist_rec_flag *flags_r,
const char **fname_r)
{
struct maildir_uidlist_rec *rec;
unsigned int idx;
int ret;
return ret;
return 1;
}
const char *
{
struct maildir_uidlist_rec *rec;
unsigned int idx;
const unsigned char *p;
int ret;
return NULL;
p = rec->extensions;
while (*p != '\0') {
/* <key><value>\0 */
if (*p == (char)key)
return (const char *)p + 1;
p += strlen((const char *)p) + 1;
}
return NULL;
}
{
return uidlist->uid_validity;
}
{
}
{
if (!uidlist->have_mailbox_guid) {
if (maildir_uidlist_update(uidlist) < 0)
return -1;
}
return 0;
}
{
sizeof(uidlist->mailbox_guid)) != 0) {
sizeof(uidlist->mailbox_guid));
}
}
{
i_assert(uid_validity != 0);
}
}
{
}
}
static void
const char *value)
{
struct maildir_uidlist_rec *rec;
unsigned int idx;
const unsigned char *p;
unsigned int len;
int ret;
if (ret <= 0) {
if (ret < 0)
return;
/* maybe it's a new message */
if (maildir_uidlist_refresh(uidlist) < 0)
return;
/* message is already expunged, ignore */
return;
}
}
/* copy existing extensions, except for the one we're updating */
p = rec->extensions;
while (*p != '\0') {
/* <key><value>\0 */
if (*p != (char)key)
p += len;
}
}
}
/* message already exists in uidlist, need to recreate it */
}
}
const char *value)
{
T_BEGIN {
} T_END;
}
static void
{
const struct mail_index_header *hdr;
if (hdr->uid_validity != 0) {
return;
}
}
}
{
struct maildir_uidlist_iter_ctx *iter;
struct maildir_uidlist_rec *rec;
const unsigned char *p;
unsigned int len;
int ret;
if (uidlist->uid_validity == 0)
if (!uidlist->have_mailbox_guid)
sizeof(uidlist->mailbox_guid)));
}
}
str_truncate(str, 0);
p += len + 1;
}
}
}
if (ret < 0) {
"o_stream_send(%s) failed: %m", path);
return -1;
}
"fdatasync(%s) failed: %m", path);
return -1;
}
}
return 0;
}
static void
{
struct mail_index_view *view;
struct maildir_uidlist_rec *const *recs;
const struct mail_index_header *hdr;
const struct mail_index_record *rec;
unsigned int i, count;
/* we could get here when opening and locking mailbox,
before index files have been opened. */
return;
/* too much trouble to be worth it */
return;
}
/* expunged entry */
i++;
/* index isn't up to date. we're probably just
syncing it here. ignore this entry. */
seq++;
} else {
seq++; i++;
}
}
/* drop messages expunged at the end of index */
i++;
}
/* view might not be completely up-to-date, so preserve any
messages left */
for (; i < count; i++)
}
{
const char *control_dir, *temp_path;
for (i = 0;; i++) {
if (fd != -1)
break;
"open(%s, O_CREAT) failed: %m", temp_path);
return -1;
}
/* the control dir doesn't exist. create it unless the whole
mailbox was just deleted. */
return -1;
}
} else {
"fchown(%s) failed: %m", temp_path);
}
}
uidlist->read_records_count = 0;
if (ret == 0) {
"rename(%s, %s) failed: %m",
ret = -1;
}
}
if (ret < 0) {
"unlink(%s) failed: %m", temp_path);
}
"fstat(%s) failed: %m", temp_path);
ret = -1;
"Maildir uidlist dotlock overridden: %s",
ret = -1;
} else {
}
if (ret < 0)
return ret;
}
{
int ret;
return 0;
if (maildir_uidlist_lock(uidlist) <= 0)
return -1;
return ret;
}
{
unsigned int min_rewrite_count, messages_count;
return FALSE;
return TRUE;
UIDLIST_COMPRESS_PERCENTAGE / 100;
return min_rewrite_count >= messages_count;
}
{
return FALSE;
return TRUE;
return TRUE;
return maildir_uidlist_want_compress(ctx);
}
{
return maildir_uidlist_recreate(uidlist);
/* make sure we have the latest file (e.g. NOREFRESH used) */
if (maildir_uidlist_open_latest(uidlist) < 0)
return -1;
}
return -1;
}
return -1;
return -1;
}
i_warning("%s: file size changed unexpectedly after write",
} else if (uidlist->locked_refresh) {
}
return 0;
}
bool nonsynced)
{
struct maildir_uidlist_rec **recs;
unsigned int i, count;
if (nonsynced) {
for (i = 0; i < count; i++)
} else {
for (i = 0; i < count; i++)
}
}
bool *locked_r)
{
int ret;
if ((sync_flags & MAILDIR_UIDLIST_SYNC_NOLOCK) != 0) {
if (maildir_uidlist_refresh(uidlist) < 0)
return -1;
return 1;
}
if (ret <= 0) {
return ret;
/* couldn't lock it */
if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0)
return 0;
/* forcing the sync anyway */
if (maildir_uidlist_refresh(uidlist) < 0)
return -1;
} else {
}
return 1;
}
{
}
struct maildir_uidlist_sync_ctx **sync_ctx_r)
{
struct maildir_uidlist_sync_ctx *ctx;
bool locked;
int ret;
if (ret <= 0)
return ret;
(sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0;
if ((sync_flags & MAILDIR_UIDLIST_SYNC_KEEP_STATE) == 0) {
/* initially mark all nonsynced */
}
return 1;
}
"maildir_uidlist_sync", 16384);
return 1;
}
static void
{
struct maildir_uidlist_rec *rec;
/* we'll update uidlist directly */
/* doesn't exist in uidlist */
/* we can't add it, so just ignore it */
return;
}
ctx->new_files_count++;
"uidlist record_pool",
1024);
}
struct maildir_uidlist_rec, 1);
}
if (uid != 0) {
}
}
{
unsigned char *ret;
if (extensions == NULL)
return NULL;
T_BEGIN {
unsigned int len;
}
} T_END;
return ret;
}
const char *filename,
{
}
{
const char *p, *dir;
return -1;
for (p = filename; *p != '\0'; p++) {
if (*p == 13 || *p == 10) {
i_warning("Maildir %s: Ignoring a file with #0x%x: %s",
return 1;
}
}
return 1;
}
MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) {
/* possibly duplicate */
return 0;
}
/* probably was in new/ and now we're seeing it in cur/.
to check for duplicates. */
} else {
rec->extensions =
} else {
ctx->new_files_count++;
/* didn't exist in uidlist, it's recent */
}
}
if (uid != 0) {
}
return 1;
}
const char *filename)
{
struct maildir_uidlist_rec *rec;
unsigned int idx;
}
ctx->first_new_pos--;
}
}
const char *
const char *filename)
{
struct maildir_uidlist_rec *rec;
}
{
struct maildir_uidlist_rec *rec;
return FALSE;
return TRUE;
}
const char *
const char *filename)
{
struct maildir_uidlist_rec *rec;
}
{
return -1;
else
return 1;
}
}
{
struct maildir_uidlist_rec **recs;
/* sort new files and assign UIDs for them */
sizeof(*recs), maildir_assign_uid_cmp);
break;
}
}
ctx->new_files_count = 0;
}
{
/* buffer is unsorted, sort it by UID */
if (ctx->new_files_count != 0) {
} else {
}
}
{
}
{
} else {
}
}
/* mbox=NULL means we're coming from dbox rebuilding code.
the dbox is already locked, so allow uidlist recreation */
T_BEGIN {
if (maildir_uidlist_sync_update(ctx) < 0) {
/* we couldn't write everything we wanted. make
sure we don't continue using those UIDs */
}
} T_END;
}
}
bool success)
{
int ret;
if (!success)
return ret;
}
const char *filename,
{
struct maildir_uidlist_rec *rec;
}
struct maildir_uidlist_iter_ctx *
{
struct maildir_uidlist_iter_ctx *ctx;
unsigned int count;
return ctx;
}
static void
{
count - old_rev_idx;
idx++;
idx--;
}
struct maildir_uidlist_rec **rec_r)
{
struct maildir_uidlist_rec *rec;
return FALSE;
return TRUE;
}
enum maildir_uidlist_rec_flag *flags_r,
const char **filename_r)
{
struct maildir_uidlist_rec *rec;
return FALSE;
return TRUE;
}
{
}