mailbox-list-index-sync.c revision 5fb3bff645380804c9db2510940c41db6b8fdb01
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny/* Copyright (C) 2006 Timo Sirainen */
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "lib.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "array.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "bsearch-insert-pos.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "crc32.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "file-set-size.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "mmap-util.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "mail-index-private.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include "mailbox-list-index-private.h"
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#include <stddef.h>
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#define ROOT_INIT_COUNT 128
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#define DIR_ALLOC_MORE_COUNT 4
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#define MAILBOX_LIST_INDEX_GROW_PERCENTAGE 10
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny#define MAILBOX_LIST_INDEX_MIN_SIZE 512
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zelenystruct mailbox_list_sync_record {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny uint32_t name_hash;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t seq;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t uid;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny const char *name;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny /* dir is used if it's non-NULL, otherwise dir_offset is used */
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny struct mailbox_list_sync_dir *dir;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t dir_offset;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t created:1;
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina uint32_t seen:1;
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina};
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březinastruct mailbox_list_sync_dir {
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina /* The records are sorted by their name_hash */
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina ARRAY_DEFINE(records, struct mailbox_list_sync_record);
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina /* Offset to the original location in the index, or 0 for new dirs */
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina uint32_t offset;
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina unsigned int seen_records_count;
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina unsigned int new_records_count;
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina};
cf1a8af5556b1d8eab68802918c881ae1a0b89ebPavel Březina
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorcestruct mailbox_list_index_sync_ctx {
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce struct mailbox_list_index *index;
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce pool_t pool;
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce enum mailbox_list_sync_flags flags;
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce const char *sync_path;
1f800ebb0f190854b8296146174f3d696a426333Simo Sorce struct mail_index_sync_ctx *mail_sync_ctx;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce struct mail_index_view *view;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce struct mail_index_transaction *trans;
1f800ebb0f190854b8296146174f3d696a426333Simo Sorce
1f800ebb0f190854b8296146174f3d696a426333Simo Sorce struct mailbox_list_index_header hdr;
07b92f78d1751d8a2a538a440e1fdb24c59021e0Pavel Březina struct mailbox_list_sync_dir *root, *sync_root;
1f800ebb0f190854b8296146174f3d696a426333Simo Sorce
1f800ebb0f190854b8296146174f3d696a426333Simo Sorce unsigned int failed:1;
850ca620611f65115ee95e1d919be8443f95c14cLukas Slebodnik unsigned int partial:1;
1f800ebb0f190854b8296146174f3d696a426333Simo Sorce unsigned int seen_sync_root:1;
850ca620611f65115ee95e1d919be8443f95c14cLukas Slebodnik};
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce
1f800ebb0f190854b8296146174f3d696a426333Simo Sorcestruct mailbox_list_sync_lookup_key {
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce uint32_t name_hash;
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce const char *name;
0232747f04b650796db56fd7b487aee8a96fab03Simo Sorce bool *match;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek};
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozekstatic struct mailbox_list_sync_dir *
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozekmailbox_list_alloc_sync_dir(struct mailbox_list_index_sync_ctx *ctx,
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek unsigned int initial_count)
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek{
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek struct mailbox_list_sync_dir *sync_dir;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek sync_dir = p_new(ctx->pool, struct mailbox_list_sync_dir, 1);
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek p_array_init(&sync_dir->records, ctx->pool, initial_count);
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek return sync_dir;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek}
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozekstatic int
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozekmailbox_list_copy_sync_dir(struct mailbox_list_index_sync_ctx *ctx,
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek uint32_t offset,
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek struct mailbox_list_sync_dir **sync_dir_r)
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek{
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek const struct mailbox_list_dir_record *dir;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek const struct mailbox_list_record *recs;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek struct mailbox_list_sync_dir *sync_dir;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek struct mailbox_list_sync_record *sync_rec;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek const char *name;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek size_t max_len;
33c865412732554ef255e93c4e7a58b0bce963c6Jakub Hrozek unsigned int i;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce if (mailbox_list_index_get_dir(ctx->index, &offset, &dir) < 0)
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce return -1;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_dir = mailbox_list_alloc_sync_dir(ctx, dir->count +
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce DIR_ALLOC_MORE_COUNT);
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_dir->offset = offset;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce recs = MAILBOX_LIST_RECORDS(dir);
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce for (i = 0; i < dir->count; i++) {
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_rec = array_append_space(&sync_dir->records);
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_rec->name_hash = recs[i].name_hash;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_rec->uid = recs[i].uid;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_rec->dir_offset =
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce mail_index_offset_to_uint32(recs[i].dir_offset);
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce max_len = ctx->index->mmap_size - recs[i].name_offset;
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce name = CONST_PTR_OFFSET(ctx->index->mmap_base,
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce recs[i].name_offset);
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce
bba1a5fd62cffcae076d1351df5a83fbc4a6ec17Simo Sorce sync_rec->name = p_strndup(ctx->pool, name, max_len);
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina }
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina *sync_dir_r = sync_dir;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina return 0;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina}
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březinastatic int mailbox_list_sync_record_cmp(const void *_key, const void *_rec)
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina{
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina const struct mailbox_list_sync_lookup_key *key = _key;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina const struct mailbox_list_sync_record *rec = _rec;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina int ret;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina if (key->name_hash < rec->name_hash)
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina return -1;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina if (key->name_hash > rec->name_hash)
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina return 1;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina ret = strcmp(key->name, rec->name);
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina if (ret == 0)
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina *key->match = TRUE;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina return ret;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina}
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březinastatic struct mailbox_list_sync_record *
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březinamailbox_list_sync_dir_lookup(struct mailbox_list_sync_dir *dir,
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina const char *name, unsigned int *idx_r)
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina{
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina struct mailbox_list_sync_lookup_key key;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina const struct mailbox_list_sync_record *recs;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina struct mailbox_list_sync_record *rec;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina unsigned int count;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina bool match;
e1f68731525116ce686ffcdc07ad3a14e4fb1cd7Pavel Březina
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina /* binary search the current hierarchy level name. the values are
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina sorted primarily by their hash value and secondarily by the actual
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina name */
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina match = FALSE;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina key.name = name;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina key.name_hash = crc32_str(name);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina key.match = &match;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina recs = array_get(&dir->records, &count);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina rec = bsearch_insert_pos(&key, recs, count, sizeof(*rec),
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina mailbox_list_sync_record_cmp);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina *idx_r = rec - recs;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina return match ? rec : NULL;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina}
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březinastatic struct mailbox_list_sync_record *
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březinamailbox_list_alloc_add_record(struct mailbox_list_index_sync_ctx *ctx,
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina struct mailbox_list_sync_dir *dir,
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina const char *name, unsigned int idx)
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina{
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina struct mailbox_list_sync_record *rec;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina rec = array_insert_space(&dir->records, idx);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina rec->name_hash = crc32_str(name);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina rec->name = p_strdup(ctx->pool, name);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina rec->uid = ctx->hdr.next_uid++;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina rec->created = TRUE;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina mail_index_append(ctx->trans, rec->uid, &rec->seq);
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina dir->new_records_count++;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina return rec;
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina}
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březina
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březinastatic int
0b81cc5d41ec6a1c58e610f402fd93a1fbda4affPavel Březinamailbox_list_index_sync_get_seq(struct mailbox_list_index_sync_ctx *ctx,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny struct mailbox_list_sync_record *rec)
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny{
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (rec->uid == 0) {
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce return mailbox_list_index_set_corrupted(ctx->index,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny "Record with UID=0");
9822d4d468ec74e4e173f5adf0db12d02974cd18Sumit Bose }
a6cca9c284724fafd670a3163812f248ba53ad97Jakub Hrozek if (mail_index_lookup_uid_range(ctx->view, rec->uid, rec->uid,
a6cca9c284724fafd670a3163812f248ba53ad97Jakub Hrozek &rec->seq, &rec->seq) < 0)
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny return -1;
f91e4aacb78d33791efcd744000597d5254dac4bSimo Sorce
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (rec->seq == 0) {
aab938c5975f0e3b85c7c79a5d718e5fefed7217Simo Sorce return mailbox_list_index_set_corrupted(ctx->index,
aab938c5975f0e3b85c7c79a5d718e5fefed7217Simo Sorce "Desync: Record expunged from mail index");
aab938c5975f0e3b85c7c79a5d718e5fefed7217Simo Sorce }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny return 0;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny}
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zelenystatic int
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zelenymailbox_list_index_sync_int(struct mailbox_list_index_sync_ctx *ctx,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny const char *name,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny struct mailbox_list_sync_dir **dir_r,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny uint32_t *seq_r)
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny{
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny const char *p, *hier_name;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny struct mailbox_list_sync_dir *dir;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny struct mailbox_list_sync_record *rec = NULL;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny unsigned int idx;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (ctx->failed)
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny return -1;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny dir = ctx->sync_root;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
d3f2fd9cb21cc10dce663a2f7d0deda07074e44eJan Zeleny t_push();
d3f2fd9cb21cc10dce663a2f7d0deda07074e44eJan Zeleny for (;;) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny p = strchr(name, ctx->index->separator);
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny hier_name = p == NULL ? name : t_strdup_until(name, p);
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (*hier_name == '\0') {
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce if (p == NULL) {
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce /* name ended with a separator */
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce break;
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce }
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce /* two separators adjacently, skip this */
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce name = p + 1;
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce continue;
b8dcd1216e5ea7065213c750a92dabfe01fa3b70Simo Sorce }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (rec != NULL) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny mail_index_update_flags(ctx->trans, rec->seq,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny MODIFY_REPLACE,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny MAILBOX_LIST_INDEX_FLAG_NONEXISTENT |
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny MAILBOX_LIST_INDEX_FLAG_CHILDREN);
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny rec = mailbox_list_sync_dir_lookup(dir, hier_name, &idx);
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (rec == NULL) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny /* new record */
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny rec = mailbox_list_alloc_add_record(ctx, dir,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny hier_name, idx);
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny } else if (rec->seq == 0) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny /* this record was copied from existing index.
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny the uid is known, but the sequence isn't. */
a6cca9c284724fafd670a3163812f248ba53ad97Jakub Hrozek if (mailbox_list_index_sync_get_seq(ctx, rec) < 0) {
58fd3aa25c5292bc67432647ab7e5059439fcc6dSimo Sorce ctx->failed = TRUE;
9822d4d468ec74e4e173f5adf0db12d02974cd18Sumit Bose break;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny *seq_r = rec->seq;
bf8cce77a35cb0a3cdb0d21fb9c39b7b6372bc11Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny /* remember that we've seen this record */
ac7a7ee3d1e138818a1ed78758f7dd3c3306a56bSumit Bose if (!rec->seen) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny rec->seen = TRUE;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny dir->seen_records_count++;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (p == NULL) {
3c60433641ce2e86b9b04778c8f8652ef0d097e4Stef Walter /* leaf */
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny break;
8ccb0de226ccb9330f5a6865de487d6f0313902dJan Zeleny }
8ccb0de226ccb9330f5a6865de487d6f0313902dJan Zeleny
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (rec->dir == NULL) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (rec->dir_offset != 0) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny if (mailbox_list_copy_sync_dir(ctx,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny rec->dir_offset,
f91e4aacb78d33791efcd744000597d5254dac4bSimo Sorce &rec->dir) < 0) {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny ctx->failed = TRUE;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny break;
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny } else {
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny rec->dir = mailbox_list_alloc_sync_dir(ctx,
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny 1 + DIR_ALLOC_MORE_COUNT);
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
20d0bc6d587f346238062df4da5edfde815e59b1Jan Zeleny }
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorce name = p + 1;
234958be042980242fff6da936af674da877c5efSimo Sorce dir = rec->dir;
234958be042980242fff6da936af674da877c5efSimo Sorce }
234958be042980242fff6da936af674da877c5efSimo Sorce t_pop();
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorce i_assert(dir != NULL);
234958be042980242fff6da936af674da877c5efSimo Sorce *dir_r = dir;
234958be042980242fff6da936af674da877c5efSimo Sorce return ctx->failed ? -1 : 0;
234958be042980242fff6da936af674da877c5efSimo Sorce}
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorcestatic int mailbox_list_index_get_root(struct mailbox_list_index_sync_ctx *ctx)
234958be042980242fff6da936af674da877c5efSimo Sorce{
234958be042980242fff6da936af674da877c5efSimo Sorce uint32_t seq;
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorce i_assert(ctx->index->mmap_size > 0);
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorce if (ctx->index->mmap_size == sizeof(*ctx->index->hdr)) {
234958be042980242fff6da936af674da877c5efSimo Sorce /* root doesn't exist in the file yet */
234958be042980242fff6da936af674da877c5efSimo Sorce ctx->root = mailbox_list_alloc_sync_dir(ctx,
234958be042980242fff6da936af674da877c5efSimo Sorce ROOT_INIT_COUNT);
234958be042980242fff6da936af674da877c5efSimo Sorce } else {
234958be042980242fff6da936af674da877c5efSimo Sorce if (mailbox_list_copy_sync_dir(ctx, sizeof(*ctx->index->hdr),
234958be042980242fff6da936af674da877c5efSimo Sorce &ctx->root) < 0)
234958be042980242fff6da936af674da877c5efSimo Sorce return -1;
234958be042980242fff6da936af674da877c5efSimo Sorce }
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorce /* keep sync_root=root until we've built the sync_root path. */
234958be042980242fff6da936af674da877c5efSimo Sorce ctx->sync_root = ctx->root;
234958be042980242fff6da936af674da877c5efSimo Sorce
234958be042980242fff6da936af674da877c5efSimo Sorce if (*ctx->sync_path != '\0') {
234958be042980242fff6da936af674da877c5efSimo Sorce if (mailbox_list_index_sync_more(ctx, ctx->sync_path, &seq) < 0)
234958be042980242fff6da936af674da877c5efSimo Sorce return -1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return mailbox_list_index_sync_int(ctx, ctx->sync_path,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek &ctx->sync_root, &seq);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekstatic int sync_init_mail_sync(struct mailbox_list_index_sync_ctx *ctx)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mail_index_sync_rec sync_rec;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek const struct mail_index_header *hdr;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (mail_index_sync_begin(ctx->index->mail_index, &ctx->mail_sync_ctx,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek &ctx->view, (uint32_t)-1, 0,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek FALSE, FALSE) < 0)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return -1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
144e7bad57748aaab395b9fb8ad136cdad864a00Lukas Slebodnik /* we should have only external transactions in here, for which we
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek don't need to do anything but write them to the index */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek while (mail_index_sync_next(ctx->mail_sync_ctx, &sync_rec) > 0)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek hdr = mail_index_get_header(ctx->view);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (hdr->uid_validity != 0) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (hdr->uid_validity != ctx->hdr.uid_validity) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return mailbox_list_index_set_corrupted(ctx->index,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek "Desync: uid_validity changed");
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->trans = mail_index_transaction_begin(ctx->view, FALSE, TRUE);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (hdr->uid_validity == 0) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek mail_index_update_header(ctx->trans,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek offsetof(struct mail_index_header, uid_validity),
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek &ctx->hdr.uid_validity, sizeof(ctx->hdr.uid_validity),
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek TRUE);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return mailbox_list_index_get_root(ctx);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekint mailbox_list_index_sync_init(struct mailbox_list_index *index,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek const char *path,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek enum mailbox_list_sync_flags flags,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_index_sync_ctx **ctx_r)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_index_sync_ctx *ctx;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek pool_t pool;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek size_t len;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek /* add separator to end of path if it isn't there */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek len = strlen(path);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (len > 0 && path[len-1] != index->separator)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek path = t_strdup_printf("%s%c", path, index->separator);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek pool = pool_alloconly_create("mailbox list index sync", 1024*32);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx = p_new(pool, struct mailbox_list_index_sync_ctx, 1);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->pool = pool;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->index = index;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->sync_path = p_strdup(pool, path);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->flags = flags;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->hdr = *index->hdr;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek /* mail index syncing acts as the only locking for us */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (sync_init_mail_sync(ctx) < 0) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek mailbox_list_index_sync_commit(&ctx);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return -1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek *ctx_r = ctx;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return 0;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekstruct mail_index_view *
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekmailbox_list_index_sync_get_view(struct mailbox_list_index_sync_ctx *ctx)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return ctx->view;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekstruct mail_index_transaction *
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekmailbox_list_index_sync_get_transaction(struct mailbox_list_index_sync_ctx *ctx)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return ctx->trans;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekint mailbox_list_index_sync_more(struct mailbox_list_index_sync_ctx *ctx,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek const char *name, uint32_t *seq_r)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_sync_dir *dir;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return mailbox_list_index_sync_int(ctx, name, &dir, seq_r);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekstatic int
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekmailbox_list_index_sync_grow(struct mailbox_list_index_sync_ctx *ctx,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t size)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_index *index = ctx->index;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uoff_t new_fsize, grow_size;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek new_fsize = ctx->hdr.used_space + size;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek grow_size = new_fsize / 100 * MAILBOX_LIST_INDEX_GROW_PERCENTAGE;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (grow_size < MAILBOX_LIST_INDEX_MIN_SIZE)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek grow_size = MAILBOX_LIST_INDEX_MIN_SIZE;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek new_fsize += grow_size;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek new_fsize &= ~(512-1);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek i_assert(new_fsize >= ctx->hdr.used_space + size);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (file_set_size(index->fd, (off_t)new_fsize) < 0) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek mailbox_list_index_set_syscall_error(index, "file_set_size()");
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return -1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return mailbox_list_index_map(index);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekstatic int
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekmailbox_list_index_sync_alloc_space(struct mailbox_list_index_sync_ctx *ctx,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t size, void **base_r,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t *base_offset_r)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek size_t pos = ctx->hdr.used_space;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek /* all allocations must be 32bit aligned */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek pos = (pos + 3) & ~3;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (pos + size > ctx->index->mmap_size) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (mailbox_list_index_sync_grow(ctx, size + 3) < 0)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return -1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek i_assert(pos + size < ctx->index->mmap_size);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek *base_offset_r = pos;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek *base_r = PTR_OFFSET(ctx->index->mmap_base, *base_offset_r);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek ctx->hdr.used_space = pos + size;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return 0;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek}
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekstatic int
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozekmailbox_list_index_sync_recreate_dir(struct mailbox_list_index_sync_ctx *ctx,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_sync_dir *sync_dir,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t offset_pos, bool partial)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek{
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_dir_record *dir, *new_dir;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_record *recs, *new_recs;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek struct mailbox_list_sync_record *sync_recs;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek unsigned int src, dest, orig, count, nondeleted_count;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek unsigned int name_space_needed, deleted_space;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek uint32_t base_offset, name_pos, size;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek void *base;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek i_assert((offset_pos % sizeof(uint32_t)) == 0);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek i_assert(offset_pos < ctx->index->mmap_size);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek /* count how much space we need and how much we wasted for deleted
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek records */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek nondeleted_count = 0; name_space_needed = 0; deleted_space = 0;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek sync_recs = array_get_modifiable(&sync_dir->records, &count);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek for (src = 0; src < count; src++) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (sync_recs[src].seen || partial) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek nondeleted_count++;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (sync_recs[src].created) {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek /* new record */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek name_space_needed +=
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek strlen(sync_recs[src].name) + 1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek } else {
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek deleted_space += sizeof(*new_recs) +
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek strlen(sync_recs[src].name) + 1;
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek }
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek /* @UNSAFE */
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek name_space_needed += sizeof(*dir) +
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek nondeleted_count * sizeof(*new_recs);
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek if (mailbox_list_index_sync_alloc_space(ctx, name_space_needed,
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek &base, &base_offset) < 0)
58dd26b1c5b60ee992dd5d1214bb168aebb42d54Jakub Hrozek return -1;
/* NOTE: any pointers to the index file may have been invalidated
as a result of growing the the memory area */
if (sync_dir->offset == 0) {
dir = NULL;
recs = NULL;
} else {
/* the offset should have been verified already to be valid */
i_assert(sync_dir->offset == offset_pos);
i_assert(sync_dir->offset < ctx->index->mmap_size);
dir = PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset);
recs = MAILBOX_LIST_RECORDS(dir);
}
new_dir = base;
new_dir->count = nondeleted_count;
new_recs = MAILBOX_LIST_RECORDS(new_dir);
name_pos = (const char *)(new_recs + nondeleted_count) -
(const char *)base;
for (src = dest = 0; src < count; src++) {
if (!sync_recs[src].seen && !partial) {
/* expunge from mail index */
uint32_t seq;
if (mail_index_lookup_uid_range(ctx->view,
sync_recs[src].uid,
sync_recs[src].uid,
&seq, &seq) < 0)
return -1;
if (seq != 0)
mail_index_expunge(ctx->trans, seq);
// FIXME: expunge also NONEXISTENT parents
continue;
}
new_recs[dest].name_hash = sync_recs[src].name_hash;
new_recs[dest].dir_offset =
mail_index_uint32_to_offset(sync_recs[src].dir_offset);
if (sync_recs[src].created) {
/* new record */
new_recs[dest].uid = sync_recs[src].uid;
new_recs[dest].name_offset = base_offset + name_pos;
size = strlen(sync_recs[src].name) + 1;
memcpy(PTR_OFFSET(base, name_pos), sync_recs[src].name,
size);
name_pos += size;
} else {
/* existing record. need to find its name_offset */
for (orig = 0; orig < dir->count; orig++) {
if (recs[orig].uid == sync_recs[src].uid)
break;
}
i_assert(orig < dir->count);
new_recs[dest].uid = sync_recs[src].uid;
new_recs[dest].name_offset = recs[orig].name_offset;
}
dest++;
}
i_assert(dest == nondeleted_count);
i_assert(name_pos == name_space_needed);
if (offset_pos == 0) {
/* we're writing the root directory */
i_assert(base_offset == sizeof(*ctx->index->hdr));
} else {
/* add a link to this newly created directory. */
uint32_t *pos;
pos = PTR_OFFSET(ctx->index->mmap_base, offset_pos);
i_assert(mail_index_offset_to_uint32(*pos) == 0);
*pos = mail_index_uint32_to_offset(base_offset);
}
sync_dir->offset = base_offset;
return 0;
}
static int
mailbox_list_index_sync_update_dir(struct mailbox_list_index_sync_ctx *ctx,
struct mailbox_list_sync_dir *sync_dir)
{
const struct mailbox_list_dir_record *dir;
struct mailbox_list_record *recs;
const struct mailbox_list_sync_record *sync_recs;
unsigned int i, count;
i_assert(sync_dir->offset != 0);
if (mailbox_list_index_get_dir(ctx->index, &sync_dir->offset, &dir) < 0)
return -1;
sync_recs = array_get(&sync_dir->records, &count);
i_assert(dir->count == count);
i_assert(sync_dir->seen_records_count < count);
recs = MAILBOX_LIST_RECORDS(dir);
for (i = 0; i < dir->count; i++) {
if (!sync_recs[i].seen)
recs[i].deleted = TRUE;
}
return 0;
}
static int
mailbox_list_index_sync_write_dir(struct mailbox_list_index_sync_ctx *ctx,
struct mailbox_list_sync_dir *sync_dir,
uint32_t offset_pos, bool partial)
{
const struct mailbox_list_dir_record *dir;
const struct mailbox_list_record *recs;
const struct mailbox_list_sync_record *sync_recs;
uint32_t child_offset_pos;
unsigned int i, j, count;
if (!ctx->seen_sync_root && ctx->sync_root == sync_dir) {
i_assert(partial);
ctx->seen_sync_root = TRUE;
partial = (ctx->flags & MAILBOX_LIST_SYNC_FLAG_PARTIAL) != 0;
}
if (sync_dir->offset != 0) {
/* point to latest dir entry's next_offset */
offset_pos = sync_dir->offset +
offsetof(struct mailbox_list_dir_record, next_offset);
}
if (sync_dir->new_records_count > 0) {
/* need to recreate the dir record */
if (mailbox_list_index_sync_recreate_dir(ctx, sync_dir,
offset_pos,
partial) < 0)
return -1;
/* NOTE: index may have been remaped here */
} else if (sync_dir->seen_records_count !=
array_count(&sync_dir->records) && !partial) {
/* just mark the records deleted */
if (mailbox_list_index_sync_update_dir(ctx, sync_dir) < 0)
return -1;
}
if (!partial && (ctx->flags & MAILBOX_LIST_SYNC_FLAG_RECURSIVE) == 0) {
/* we're doing a full sync only for the root */
partial = TRUE;
}
/* update child mailboxes */
sync_recs = array_get(&sync_dir->records, &count);
if (count == 0)
return 0;
i_assert(sync_dir->offset != 0 &&
sync_dir->offset < ctx->index->mmap_size);
for (i = j = 0; i < count; i++) {
if (sync_recs[i].dir == NULL)
continue;
/* these may change after each sync_write_dir() call */
dir = CONST_PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset);
recs = MAILBOX_LIST_RECORDS(dir);
/* child_offset_pos needs to point to record's dir_offset */
for (; j < dir->count; j++) {
if (recs[j].uid == sync_recs[i].uid)
break;
}
i_assert(j < dir->count);
child_offset_pos = (const char *)&recs[j].dir_offset -
(const char *)ctx->index->mmap_base;
if (mailbox_list_index_sync_write_dir(ctx, sync_recs[i].dir,
child_offset_pos,
partial) < 0)
return -1;
}
return 0;
}
static int
mailbox_list_index_sync_write(struct mailbox_list_index_sync_ctx *ctx)
{
struct mailbox_list_index_header *hdr;
bool partial;
if (ctx->sync_root == ctx->root) {
ctx->seen_sync_root = TRUE;
partial = (ctx->flags & MAILBOX_LIST_SYNC_FLAG_PARTIAL) != 0;
} else {
/* until we've seen the sync root, we're doing only partial
syncing */
partial = TRUE;
}
if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0)
return -1;
/* update header */
hdr = ctx->index->mmap_base;
hdr->next_uid = ctx->hdr.next_uid;
hdr->used_space = ctx->hdr.used_space;
hdr->deleted_space = ctx->hdr.deleted_space;
if (msync(ctx->index->mmap_base, hdr->used_space, MS_SYNC) < 0) {
mailbox_list_index_set_syscall_error(ctx->index, "msync()");
return -1;
}
return 0;
}
int mailbox_list_index_sync_commit(struct mailbox_list_index_sync_ctx **_ctx)
{
struct mailbox_list_index_sync_ctx *ctx = *_ctx;
int ret = ctx->failed ? -1 : 0;
*_ctx = NULL;
if (!ctx->failed) {
/* write all the changes to the index */
ret = mailbox_list_index_sync_write(ctx);
}
if (ctx->mail_sync_ctx != NULL) {
if (ret < 0)
mail_index_transaction_rollback(&ctx->trans);
else {
uint32_t seq;
uoff_t offset;
if (mail_index_transaction_commit(&ctx->trans,
&seq, &offset) < 0)
ret = -1;
}
if (ret < 0)
mail_index_sync_rollback(&ctx->mail_sync_ctx);
else {
if (mail_index_sync_commit(&ctx->mail_sync_ctx) < 0)
ret = -1;
}
}
pool_unref(ctx->pool);
return ret;
}
void mailbox_list_index_sync_rollback(struct mailbox_list_index_sync_ctx **ctx)
{
(*ctx)->failed = TRUE;
(void)mailbox_list_index_sync_commit(ctx);
}