dbox-file.h revision 3c89294a59c9c12743d4dc57d8ed3831e0e209ec
2454dfa32c93c20a8522c6ed42fe057baaac9f9aStephan Bosch#ifndef DBOX_FILE_H
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#define DBOX_FILE_H
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen/* The file begins with a header followed by zero or more messages:
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen <dbox message header>
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen <LF>
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen <message body>
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen <metadata>
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen Metadata block begins with DBOX_MAGIC_POST, followed by zero or more lines
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen in format <key character><value><LF>. The block ends with an empty line.
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen Unknown metadata should be ignored, but preserved when copying.
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen There should be no duplicates for the current metadata, but future
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen extensions may need them so they should be preserved.
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen*/
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#define DBOX_VERSION 2
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#define DBOX_MAGIC_PRE "\001\002"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#define DBOX_MAGIC_POST "\n\001\003\n"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen/* prefer flock(). fcntl() locking currently breaks if trying to access the
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen same file from multiple mail_storages within same process. that's why we
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen fallback to dotlocks. */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen#ifdef HAVE_FLOCK
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen# define DBOX_FILE_LOCK_METHOD_FLOCK
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen#endif
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen
072f06b60d69fe9456c3fffe20e72a7d09f2825dTimo Sirainenstruct dbox_file;
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainenstruct stat;
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainenenum dbox_header_key {
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen /* Must be sizeof(struct dbox_message_header) when appending (hex) */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen DBOX_HEADER_MSG_HEADER_SIZE = 'M',
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen /* Creation UNIX timestamp (hex) */
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen DBOX_HEADER_CREATE_STAMP = 'C',
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* metadata used by old Dovecot versions */
d979c1179d55ad86e40f869e48ef3e4db9c817b5Timo Sirainen DBOX_HEADER_OLDV1_APPEND_OFFSET = 'A'
d979c1179d55ad86e40f869e48ef3e4db9c817b5Timo Sirainen};
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainenenum dbox_metadata_key {
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* Globally unique identifier for the message. Preserved when
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen copying. */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen DBOX_METADATA_GUID = 'G',
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* POP3 UIDL overriding the default format */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen DBOX_METADATA_POP3_UIDL = 'P',
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen /* Received UNIX timestamp in hex */
0eef579fa2714c8b6b6bc2e54c545d291e657f8fTimo Sirainen DBOX_METADATA_RECEIVED_TIME = 'R',
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen /* Physical message size in hex. Necessary only if it differs from
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen the dbox_message_header.message_size_hex, for example because the
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen message is compressed. */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen DBOX_METADATA_PHYSICAL_SIZE = 'Z',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen /* Virtual message size in hex (line feeds counted as CRLF) */
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_VIRTUAL_SIZE = 'V',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen /* Pointer to external message data. Format is:
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen 1*(<start offset> <byte count> <options> <ref>) */
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_EXT_REF = 'X',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen /* Mailbox name where this message was originally saved to.
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen When rebuild finds a message whose mailbox is unknown, it's
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen placed to this mailbox. */
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_ORIG_MAILBOX = 'B',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen /* metadata used by old Dovecot versions */
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_OLDV1_EXPUNGED = 'E',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_OLDV1_FLAGS = 'F',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_OLDV1_KEYWORDS = 'K',
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen DBOX_METADATA_OLDV1_SAVE_TIME = 'S',
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen DBOX_METADATA_OLDV1_SPACE = ' '
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen};
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainenenum dbox_message_type {
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen /* Normal message */
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen DBOX_MESSAGE_TYPE_NORMAL = 'N'
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen};
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainenstruct dbox_message_header {
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen unsigned char magic_pre[2];
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned char type;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned char space1;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned char oldv1_uid_hex[8];
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen unsigned char space2;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned char message_size_hex[16];
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen /* <space reserved for future extensions, LF is always last> */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen unsigned char save_lf;
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen};
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainenstruct dbox_metadata_header {
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen unsigned char magic_post[sizeof(DBOX_MAGIC_POST)-1];
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen};
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainenstruct dbox_file {
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainen struct dbox_storage *storage;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen int refcount;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen time_t create_time;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen unsigned int file_version;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen unsigned int file_header_size;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen unsigned int msg_header_size;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen const char *cur_path;
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen char *primary_path, *alt_path;
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen int fd;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen struct istream *input;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#ifdef DBOX_FILE_LOCK_METHOD_FLOCK
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen struct file_lock *lock;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen#else
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen struct dotlock *lock;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen#endif
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen uoff_t cur_offset;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen uoff_t cur_physical_size;
8eba883232f80178b60fa416f73292bf5f990fecTimo Sirainen
8eba883232f80178b60fa416f73292bf5f990fecTimo Sirainen /* Metadata for the currently seeked metadata block. */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen pool_t metadata_pool;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen ARRAY_DEFINE(metadata, const char *);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen uoff_t metadata_read_offset;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned int appending:1;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned int deleted:1;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen unsigned int corrupted:1;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen};
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenstruct dbox_file_append_context {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen struct dbox_file *file;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen uoff_t first_append_offset, last_checkpoint_offset, last_flush_offset;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen struct ostream *output;
44320b37d20bb75e0d433423318db2f4e29d5b91Timo Sirainen};
44320b37d20bb75e0d433423318db2f4e29d5b91Timo Sirainen
44320b37d20bb75e0d433423318db2f4e29d5b91Timo Sirainen#define dbox_file_is_open(file) ((file)->fd != -1)
44320b37d20bb75e0d433423318db2f4e29d5b91Timo Sirainen#define dbox_file_is_in_alt(file) ((file)->cur_path == (file)->alt_path)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenvoid dbox_file_init(struct dbox_file *file);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenvoid dbox_file_unref(struct dbox_file **file);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen/* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error.
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen If file is deleted, deleted_r=TRUE and 1 is returned. */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenint dbox_file_open(struct dbox_file *file, bool *deleted_r);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen/* Try to open file only from primary path. */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainenint dbox_file_open_primary(struct dbox_file *file, bool *notfound_r);
44320b37d20bb75e0d433423318db2f4e29d5b91Timo Sirainen/* Close the file handle from the file, but don't free it. */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenvoid dbox_file_close(struct dbox_file *file);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen/* fstat() or stat() the file. If file is already deleted, fails with
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen errno=ENOENT. */
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainenint dbox_file_stat(struct dbox_file *file, struct stat *st_r);
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone
4182d8cd818e76856a5a1e25b343fe5ddf69fd8eTimo Sirainen else, -1 if error. */
18a41cbd38f83429b790414c1159c097af4a59b8Timo Sirainenint dbox_file_try_lock(struct dbox_file *file);
c5f46e7c1c135f89627c6f1676d6e33b35ecb80aAki Tuomivoid dbox_file_unlock(struct dbox_file *file);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen/* Seek to given offset in file. Returns 1 if ok/expunged, 0 if file/offset is
corrupted, -1 if I/O error. */
int dbox_file_seek(struct dbox_file *file, uoff_t offset);
/* Start seeking at the beginning of the file. */
void dbox_file_seek_rewind(struct dbox_file *file);
/* Seek to next message after current one. If there are no more messages,
returns 0 and last_r is set to TRUE. Returns 1 if ok, 0 if file is
corrupted, -1 if I/O error. */
int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r);
/* Start appending to dbox file */
struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file);
/* Finish writing appended mails. */
int dbox_file_append_commit(struct dbox_file_append_context **ctx);
/* Truncate appended mails. */
void dbox_file_append_rollback(struct dbox_file_append_context **ctx);
/* Get output stream for appending a new message. Returns 1 if ok, 0 if file
can't be appended to (old file version or corruption) or -1 if error. */
int dbox_file_get_append_stream(struct dbox_file_append_context *ctx,
struct ostream **output_r);
/* Call after message has been fully saved. If this isn't done, the writes
since the last checkpoint are truncated. */
void dbox_file_append_checkpoint(struct dbox_file_append_context *ctx);
/* Flush output buffer. */
int dbox_file_append_flush(struct dbox_file_append_context *ctx);
/* Read current message's metadata. Returns 1 if ok, 0 if metadata is
corrupted, -1 if I/O error. */
int dbox_file_metadata_read(struct dbox_file *file);
/* Return wanted metadata value, or NULL if not found. */
const char *dbox_file_metadata_get(struct dbox_file *file,
enum dbox_metadata_key key);
/* Returns DBOX_METADATA_PHYSICAL_SIZE if set, otherwise physical size from
header. They differ only for e.g. compressed mails. */
uoff_t dbox_file_get_plaintext_size(struct dbox_file *file);
/* Fix a broken dbox file by rename()ing over it with a fixed file. Everything
before start_offset is assumed to be valid and is simply copied. The file
is reopened afterwards. Returns 0 if ok, -1 if I/O error. */
int dbox_file_fix(struct dbox_file *file, uoff_t start_offset);
/* Delete the given dbox file. Returns 1 if deleted, 0 if file wasn't found
or -1 if error. */
int dbox_file_unlink(struct dbox_file *file);
/* Fill dbox_message_header with given size. */
void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
uoff_t message_size);
void dbox_file_set_syscall_error(struct dbox_file *file, const char *function);
void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...)
ATTR_FORMAT(2, 3);
/* private: */
const char *dbox_generate_tmp_filename(void);
void dbox_file_free(struct dbox_file *file);
int dbox_file_header_write(struct dbox_file *file, struct ostream *output);
int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r);
int dbox_file_metadata_skip_header(struct dbox_file *file);
#endif