mail-index.c revision e9128d6cb8a6ffef71e8cbb049a97c0e986f4279
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include "array.h"
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen#include "buffer.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "hash.h"
5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen#include "mmap-util.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "nfs-workarounds.h"
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include "read-full.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "write-full.h"
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include "mail-index-private.h"
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include "mail-index-sync-private.h"
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include "mail-transaction-log.h"
3ed2d0f6b5e67e2663d44489d9da3176823789a8Timo Sirainen#include "mail-cache.h"
65f8fb656051f1059f7b5a2da9c5555adcc30439Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include <stdio.h>
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include <stddef.h>
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include <time.h>
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include <sys/stat.h>
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenunsigned int mail_index_module_id = 0;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenstatic void mail_index_create_in_memory(struct mail_index *index,
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen const struct mail_index_header *hdr);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
f7539a17ea306191b53b8f5e752e228937df9ec3Timo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen{
2dd39e478269d6fb0bb26d12b394aa30ee965e38Timo Sirainen struct mail_index *index;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen index = i_new(struct mail_index, 1);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen index->dir = i_strdup(dir);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen index->prefix = i_strdup(prefix);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen index->fd = -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen index->extension_pool =
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen p_array_init(&index->extensions, index->extension_pool, 5);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen i_array_init(&index->sync_lost_handlers, 4);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen array_create(&index->mail_index_module_contexts, default_pool,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen sizeof(void *), I_MIN(5, mail_index_module_id));
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
3e564425db51f3921ce4de11859777135fdedd15Timo Sirainen index->mode = 0600;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen index->gid = (gid_t)-1;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->keywords_ext_id =
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen mail_index_ext_register(index, "keywords", 128, 2, 1);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->keywords_pool = pool_alloconly_create("keywords", 512);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen i_array_init(&index->keywords, 16);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->keywords_hash =
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen hash_create(default_pool, index->keywords_pool, 0,
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen strcase_hash, (hash_cmp_callback_t *)strcasecmp);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen return index;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen}
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainenvoid mail_index_free(struct mail_index **_index)
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen{
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen struct mail_index *index = *_index;
dfaefeabae939803ceb8c503101e86b5496541d1Timo Sirainen
dfaefeabae939803ceb8c503101e86b5496541d1Timo Sirainen *_index = NULL;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen mail_index_close(index);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen hash_destroy(index->keywords_hash);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen pool_unref(index->extension_pool);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen pool_unref(index->keywords_pool);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
57a8c6a95e4bce3eeaba36985adb81c07dd683ffTimo Sirainen array_free(&index->sync_lost_handlers);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen array_free(&index->keywords);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen array_free(&index->mail_index_module_contexts);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_free(index->error);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_free(index->dir);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen i_free(index->prefix);
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen i_free(index);
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen}
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen mode_t mode, gid_t gid)
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen{
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen index->mode = mode & 0666;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen index->gid = gid;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen}
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen uint32_t default_hdr_size,
f1743785713e7632459d623d5df2108f4b93accbTimo Sirainen uint16_t default_record_size,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uint16_t default_record_align)
70ead6466f9baa8294e71fc2fba0a4f54f488b5eTimo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen const struct mail_index_registered_ext *extensions;
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen struct mail_index_registered_ext rext;
8d630c15a8ed6f85553467c3a231a273defca5f6Timo Sirainen unsigned int i, ext_count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
ee116df08d0fdab703483e18fe8076b2ef9fd9d7Timo Sirainen extensions = array_get(&index->extensions, &ext_count);
c5ab90cfad9cc3e33bcb1baeb30ffc82a7b7053aTimo Sirainen
c5ab90cfad9cc3e33bcb1baeb30ffc82a7b7053aTimo Sirainen /* see if it's already there */
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen for (i = 0; i < ext_count; i++) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (strcmp(extensions[i].name, name) == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return i;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen memset(&rext, 0, sizeof(rext));
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen rext.name = p_strdup(index->extension_pool, name);
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen rext.index_idx = ext_count;
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen rext.hdr_size = default_hdr_size;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen rext.record_size = default_record_size;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen rext.record_align = default_record_align;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen array_append(&index->extensions, &rext, 1);
cff1f182205e674285cf3ff446a0dcf7afea277dTimo Sirainen return ext_count;
cff1f182205e674285cf3ff446a0dcf7afea277dTimo Sirainen}
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainenvoid mail_index_register_expunge_handler(struct mail_index *index,
e03d986a74128f5ba30fcfda9f6e36578f5d8decTimo Sirainen uint32_t ext_id, bool call_always,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_expunge_handler_t *cb,
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen void *context)
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen{
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen struct mail_index_registered_ext *rext;
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb);
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen
d3280fe317a4598c0868cc440e7a1191c06d0db3Timo Sirainen rext->expunge_handler = cb;
d3280fe317a4598c0868cc440e7a1191c06d0db3Timo Sirainen rext->expunge_context = context;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen rext->expunge_handler_call_always = call_always;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen}
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenvoid mail_index_unregister_expunge_handler(struct mail_index *index,
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen uint32_t ext_id)
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen{
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen struct mail_index_registered_ext *rext;
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen i_assert(rext->expunge_handler != NULL);
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen rext->expunge_handler = NULL;
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen}
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainenvoid mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id,
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen mail_index_sync_handler_t *cb,
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen enum mail_index_sync_handler_type type)
c979eeda1f46483d9c963e265786b701d7683d77Timo Sirainen{
c979eeda1f46483d9c963e265786b701d7683d77Timo Sirainen struct mail_index_registered_ext *rext;
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen
2584e86cc2d8c31ba30a4109cf4ba09d1e37e28aTimo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen i_assert(rext->sync_handler.callback == NULL);
4b41116563110d00330896a568eff1078c382827Timo Sirainen
4b41116563110d00330896a568eff1078c382827Timo Sirainen rext->sync_handler.callback = cb;
4b41116563110d00330896a568eff1078c382827Timo Sirainen rext->sync_handler.type = type;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen}
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainenvoid mail_index_unregister_sync_handler(struct mail_index *index,
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen uint32_t ext_id)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_registered_ext *rext;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen rext = array_idx_modifiable(&index->extensions, ext_id);
b2c1349cf07410aefab0f5b17153af9e5cfcf48fTimo Sirainen i_assert(rext->sync_handler.callback != NULL);
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen rext->sync_handler.callback = NULL;
dbe64f3893616a4005c8946be75d2dc8f6823d72Timo Sirainen rext->sync_handler.type = 0;
8a13b020f90e080570658b18c042e7e352c8b14fTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenvoid mail_index_register_sync_lost_handler(struct mail_index *index,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mail_index_sync_lost_handler_t *cb)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen{
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen array_append(&index->sync_lost_handlers, &cb, 1);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenvoid mail_index_unregister_sync_lost_handler(struct mail_index *index,
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen mail_index_sync_lost_handler_t *cb)
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen{
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen mail_index_sync_lost_handler_t *const *handlers;
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen unsigned int i, count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f3bb2fbe87425dc89a839908985af496f7f65702Timo Sirainen handlers = array_get(&index->sync_lost_handlers, &count);
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen for (i = 0; i < count; i++) {
a3ee5ce6ecc8e228ee69300fdd562d7ac8be89a7Timo Sirainen if (handlers[i] == cb) {
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainen array_delete(&index->sync_lost_handlers, i, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen break;
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen }
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen }
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen}
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen unsigned int initial_count)
bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#define EXTENSION_NAME_APPROX_LEN 20
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen#define EXT_GLOBAL_ALLOC_SIZE \
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen ((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen#define EXT_PER_ALLOC_SIZE \
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen (EXTENSION_NAME_APPROX_LEN + \
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen sizeof(struct mail_index_ext) + sizeof(uint32_t))
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen size_t size;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen if (map->extension_pool == NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size = EXT_GLOBAL_ALLOC_SIZE +
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen initial_count * EXT_PER_ALLOC_SIZE;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen map->extension_pool =
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen pool_alloconly_create("map extensions",
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen nearest_power(size));
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen } else {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen p_clear(map->extension_pool);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen /* try to use the existing pool's size for initial_count so
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen we don't grow it unneededly */
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen size = p_get_max_easy_alloc_size(map->extension_pool);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen EXT_PER_ALLOC_SIZE;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen p_array_init(&map->extensions, map->extension_pool, initial_count);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen}
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainenuint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen{
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen const struct mail_index_ext *extensions;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen unsigned int i, size;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen if (!array_is_created(&map->extensions))
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen return (uint32_t)-1;
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen extensions = array_get(&map->extensions, &size);
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen for (i = 0; i < size; i++) {
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen if (strcmp(extensions[i].name, name) == 0)
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen return i;
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen }
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen return (uint32_t)-1;
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen}
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
42507d758b053bb483de58fba55c73a9eb5d3fbaTimo Sirainenuint32_t
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainenmail_index_map_register_ext(struct mail_index *index,
42507d758b053bb483de58fba55c73a9eb5d3fbaTimo Sirainen struct mail_index_map *map, const char *name,
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen uint32_t hdr_offset, uint32_t hdr_size,
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen uint32_t record_offset, uint32_t record_size,
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen uint32_t record_align, uint32_t reset_id)
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen{
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen struct mail_index_ext *ext;
fcfe85637e1ee14a9fc39c41fd6ceca106301542Timo Sirainen uint32_t idx, empty_idx = (uint32_t)-1;
fcfe85637e1ee14a9fc39c41fd6ceca106301542Timo Sirainen
fcfe85637e1ee14a9fc39c41fd6ceca106301542Timo Sirainen if (!array_is_created(&map->extensions)) {
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen mail_index_map_init_extbufs(map, 5);
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen idx = 0;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen } else {
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen idx = array_count(&map->extensions);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1);
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen ext = array_append_space(&map->extensions);
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen ext->name = p_strdup(map->extension_pool, name);
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen ext->hdr_offset = hdr_offset;
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen ext->hdr_size = hdr_size;
7baab0b0b60df7ce9093d0881cd322dff1e79491Timo Sirainen ext->record_offset = record_offset;
7baab0b0b60df7ce9093d0881cd322dff1e79491Timo Sirainen ext->record_size = record_size;
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen ext->record_align = record_align;
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen ext->reset_id = reset_id;
6b0d8106ae51ffc6ce45636b34d2e21cbefca7fdTimo Sirainen
6b0d8106ae51ffc6ce45636b34d2e21cbefca7fdTimo Sirainen ext->index_idx = mail_index_ext_register(index, name, hdr_size,
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen record_size, record_align);
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen /* Update index ext_id -> map ext_id mapping. Fill non-used
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen ext_ids with (uint32_t)-1 */
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen while (array_count(&map->ext_id_map) < ext->index_idx)
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen array_append(&map->ext_id_map, &empty_idx, 1);
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen return idx;
b3484b5b1f47e4cf112f0e371478a2d7794b31bbTimo Sirainen}
b3484b5b1f47e4cf112f0e371478a2d7794b31bbTimo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenstatic bool size_check(size_t *size_left, size_t size)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen{
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (size > *size_left)
0d86aa0d47f7393c669c084b34c0537b193688adTimo Sirainen return FALSE;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen *size_left -= size;
7631f16156aca373004953fe6b01a7f343fb47e0Timo Sirainen return TRUE;
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen}
aa247243412a49f9bdebf7255e131dc6ece4ed46Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainenstatic size_t get_align(size_t name_len)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen{
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen size_t size = sizeof(struct mail_index_ext_header) + name_len;
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen}
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainenstatic int mail_index_parse_extensions(struct mail_index *index,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen struct mail_index_map *map)
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen{
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen const struct mail_index_ext_header *ext_hdr;
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen unsigned int i, old_count;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen const char *name;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen uint32_t ext_id, offset, name_offset;
c0225f7f6b43d34dc58c17d3304f0fd60ab89894Timo Sirainen size_t size_left;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen /* extension headers always start from 64bit offsets, so if base header
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen doesn't happen to be 64bit aligned we'll skip some bytes */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen /* nothing to do, skip allocatations and all */
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 1;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen old_count = array_count(&index->extensions);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ext_id = (uint32_t)-1;
6df0ab0c1ab91f06b6418cb30eff44405a1b8f02Timo Sirainen for (i = 0; i < old_count; i++)
6df0ab0c1ab91f06b6418cb30eff44405a1b8f02Timo Sirainen array_append(&map->ext_id_map, &ext_id, 1);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen while (offset < map->hdr.header_size) {
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen /* Extension header contains:
9af6cc9ebc9986c1275ebdfa29c39e152af1557eTimo Sirainen - struct mail_index_ext_header
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen - name (not 0-terminated)
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen - 64bit alignment padding
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen - extension header contents
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen - 64bit alignment padding
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen */
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen size_left = map->hdr.header_size - offset;
225e82df5dd1e765f4e52b80c954558f00e5a7dfTimo Sirainen if (!size_check(&size_left, sizeof(*ext_hdr)) ||
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen !size_check(&size_left, ext_hdr->name_size) ||
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen !size_check(&size_left, get_align(ext_hdr->name_size)) ||
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen !size_check(&size_left, ext_hdr->hdr_size)) {
6564208826b0f46a00f010d1b5711d85944c3c88Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen "Header extension goes outside header",
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen index->filepath);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return -1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen offset += sizeof(*ext_hdr);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen name_offset = offset;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen t_push();
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen ext_hdr->name_size);
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) {
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen "Duplicate header extension %s",
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->filepath, name);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen t_pop();
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e4c90f0b88e40a8f92b8f5e1f1a3ea701e5c965cTimo Sirainen if (map->hdr.record_size <
defb12ecd360df672ffb2f4dbf4d1218a0a9549cTimo Sirainen ext_hdr->record_offset + ext_hdr->record_size) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen "Record field %s points outside record size "
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen "(%u < %u+%u)", index->filepath, name,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen map->hdr.record_size,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen ext_hdr->record_offset, ext_hdr->record_size);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen t_pop();
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen return -1;
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen }
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 ||
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen (map->hdr.record_size % ext_hdr->record_align) != 0) {
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen "Record field %s alignmentation %u not used",
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen index->filepath, name, ext_hdr->record_align);
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen t_pop();
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen return -1;
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen }
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen mail_index_map_register_ext(index, map, name,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen offset, ext_hdr->hdr_size,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext_hdr->record_offset,
194603b35061fea1ee8d171a7104b6985c610966Timo Sirainen ext_hdr->record_size,
194603b35061fea1ee8d171a7104b6985c610966Timo Sirainen ext_hdr->record_align,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen ext_hdr->reset_id);
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen t_pop();
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen }
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen return 1;
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen}
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainenbool mail_index_keyword_lookup(struct mail_index *index,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen const char *keyword, bool autocreate,
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen unsigned int *idx_r)
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen{
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen char *keyword_dup;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen void *value;
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen /* keywords_hash keeps a name => index mapping of keywords.
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen Keywords are never removed from it, so the index values are valid
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen for the lifetime of the mail_index. */
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) {
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen *idx_r = POINTER_CAST_TO(value, unsigned int);
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen return TRUE;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (!autocreate) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen *idx_r = (unsigned int)-1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return FALSE;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen keyword = keyword_dup = p_strdup(index->keywords_pool, keyword);
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen *idx_r = array_count(&index->keywords);
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r));
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen array_append(&index->keywords, &keyword, 1);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return TRUE;
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen}
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainenint mail_index_map_parse_keywords(struct mail_index *index,
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen struct mail_index_map *map)
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen{
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen const struct mail_index_ext *ext;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen const struct mail_index_keyword_header *kw_hdr;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen const struct mail_index_keyword_header_rec *kw_rec;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen const char *name;
a3ee5ce6ecc8e228ee69300fdd562d7ac8be89a7Timo Sirainen unsigned int i, name_area_end_offset, old_count;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen uint32_t ext_id;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen map->keywords_read = TRUE;
4321f6c969e7b8f6b243ff5bb6b8d297921676f6Timo Sirainen
4321f6c969e7b8f6b243ff5bb6b8d297921676f6Timo Sirainen ext_id = mail_index_map_lookup_ext(map, "keywords");
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen if (ext_id == (uint32_t)-1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (array_is_created(&map->keyword_idx_map))
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen array_clear(&map->keyword_idx_map);
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen return 0;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen ext = array_idx(&map->extensions, ext_id);
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen /* Extension header contains:
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen - struct mail_index_keyword_header
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen - struct mail_index_keyword_header_rec * keywords_count
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen - const char names[] * keywords_count
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen */
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen i_assert(ext->hdr_offset < map->hdr.header_size);
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen kw_rec = (const void *)(kw_hdr + 1);
e050e5c9b1688765f1fdfce9b7141f7b614383fdTimo Sirainen name = (const char *)(kw_rec + kw_hdr->keywords_count);
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen array_count(&map->keyword_idx_map);
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen /* Keywords can only be added into same mapping. Removing requires a
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen new mapping (recreating the index file) */
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen if (kw_hdr->keywords_count == old_count) {
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen /* nothing changed */
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen return 0;
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen }
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen /* make sure the header is valid */
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen if (kw_hdr->keywords_count < old_count) {
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen "Keywords removed unexpectedly",
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen index->filepath);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen
de58be41126e5d68008d2ea706d62ccdc1f29337Timo Sirainen if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen "keywords_count larger than header size",
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen index->filepath);
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen for (i = 0; i < kw_hdr->keywords_count; i++) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (kw_rec[i].name_offset > name_area_end_offset) {
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen "name_offset points outside allocated header",
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->filepath);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return -1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen if (name[name_area_end_offset-1] != '\0') {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen "Keyword header doesn't end with NUL",
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen index->filepath);
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen return -1;
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* create file -> index mapping */
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (!array_is_created(&map->keyword_idx_map))
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#ifdef DEBUG
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen /* Check that existing headers are still the same. It's behind DEBUG
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen since it's pretty useless waste of CPU normally. */
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen const unsigned int *old_idx;
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen unsigned int idx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen old_idx = array_idx(&map->keyword_idx_map, i);
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) ||
ebe6df72f1309135f02b6a4d2aef1e81a073f91cTimo Sirainen idx != *old_idx) {
910fa4e4204a73d3d24c03f3059dd24e727ca057Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
7631f16156aca373004953fe6b01a7f343fb47e0Timo Sirainen "Keywords changed unexpectedly",
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen index->filepath);
b83deefd2cf1e293373673eefb4d5cf60907978cTimo Sirainen return -1;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen }
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen }
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen#endif
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen /* Register the newly seen keywords */
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen i = array_count(&map->keyword_idx_map);
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen for (; i < kw_hdr->keywords_count; i++) {
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen const char *keyword = name + kw_rec[i].name_offset;
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen unsigned int idx;
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen array_append(&map->keyword_idx_map, &idx, 1);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen }
b3b4f3875850099c9292ad74d08bb385c3988f8fTimo Sirainen return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
const ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index)
{
/* Make sure all the keywords are in index->keywords. It's quick to do
if nothing has changed. */
(void)mail_index_map_parse_keywords(index, index->map);
return &index->keywords;
}
static int mail_index_check_header(struct mail_index *index,
struct mail_index_map *map)
{
const struct mail_index_header *hdr = &map->hdr;
enum mail_index_header_compat_flags compat_flags = 0;
#ifndef WORDS_BIGENDIAN
compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently(?) */
return -1;
}
if (hdr->compat_flags != compat_flags) {
/* architecture change - handle silently(?) */
return -1;
}
if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
/* we've already complained about it */
return -1;
}
/* following some extra checks that only take a bit of CPU */
if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
mail_index_set_error(index, "Corrupted index file %s: "
"uid_validity = 0, next_uid = %u",
index->filepath, hdr->next_uid);
return -1;
}
if (hdr->record_size < sizeof(struct mail_index_record)) {
mail_index_set_error(index, "Corrupted index file %s: "
"record_size too small: %u < %"PRIuSIZE_T,
index->filepath, hdr->record_size,
sizeof(struct mail_index_record));
return -1;
}
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0)
return 0;
if (hdr->next_uid == 0)
return 0;
if (hdr->recent_messages_count > hdr->messages_count ||
hdr->seen_messages_count > hdr->messages_count ||
hdr->deleted_messages_count > hdr->messages_count)
return 0;
if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
hdr->first_unseen_uid_lowwater > hdr->next_uid ||
hdr->first_deleted_uid_lowwater > hdr->next_uid)
return 0;
if (map->records_count > 0) {
/* last message's UID must be smaller than next_uid.
also make sure it's not zero. */
const struct mail_index_record *rec;
rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);
if (rec->uid == 0 || rec->uid >= hdr->next_uid)
return 0;
}
return mail_index_parse_extensions(index, map);
}
static void mail_index_map_clear(struct mail_index *index,
struct mail_index_map *map)
{
if (map->buffer != NULL) {
i_assert(map->mmap_base == NULL);
buffer_free(map->buffer);
map->buffer = NULL;
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
if (map->refcount > 0) {
memset(&map->hdr, 0, sizeof(map->hdr));
map->mmap_size = 0;
map->mmap_used_size = 0;
map->records = NULL;
map->records_count = 0;
}
}
void mail_index_unmap(struct mail_index *index, struct mail_index_map **_map)
{
struct mail_index_map *map = *_map;
*_map = NULL;
if (--map->refcount > 0)
return;
i_assert(map->refcount == 0);
mail_index_map_clear(index, map);
if (map->extension_pool != NULL)
pool_unref(map->extension_pool);
if (array_is_created(&map->keyword_idx_map))
array_free(&map->keyword_idx_map);
buffer_free(map->hdr_copy_buf);
i_free(map);
}
static void mail_index_map_copy_hdr(struct mail_index_map *map,
const struct mail_index_header *hdr)
{
if (hdr->base_header_size < sizeof(map->hdr)) {
/* header smaller than ours, make a copy so our newer headers
won't have garbage in them */
memset(&map->hdr, 0, sizeof(map->hdr));
memcpy(&map->hdr, hdr, hdr->base_header_size);
} else {
map->hdr = *hdr;
}
}
static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
{
const struct mail_index_header *hdr;
unsigned int records_count;
i_assert(!map->write_to_disk);
if (map->buffer != NULL) {
/* we had temporarily used a buffer, eg. for updating index */
buffer_free(map->buffer);
map->buffer = NULL;
}
map->mmap_base = index->readonly ?
mmap_ro_file(index->fd, &map->mmap_size) :
mmap_rw_file(index->fd, &map->mmap_size);
if (map->mmap_base == MAP_FAILED) {
map->mmap_base = NULL;
mail_index_set_syscall_error(index, "mmap()");
return -1;
}
hdr = map->mmap_base;
if (map->mmap_size >
offsetof(struct mail_index_header, major_version) &&
hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
mail_index_set_error(index, "Corrupted index file %s: "
"File too small (%"PRIuSIZE_T")",
index->filepath, map->mmap_size);
return 0;
}
map->mmap_used_size = hdr->header_size +
hdr->messages_count * hdr->record_size;
if (map->mmap_used_size > map->mmap_size) {
records_count = (map->mmap_size - hdr->header_size) /
hdr->record_size;
mail_index_set_error(index, "Corrupted index file %s: "
"messages_count too large (%u > %u)",
index->filepath, hdr->messages_count,
records_count);
return 0;
}
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->mmap_base;
map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
map->records_count = map->hdr.messages_count;
return 1;
}
static int mail_index_read_header(struct mail_index *index,
void *buf, size_t buf_size, size_t *pos_r)
{
size_t pos;
int ret;
memset(buf, 0, sizeof(struct mail_index_header));
/* try to read the whole header, but it's not necessarily an error to
read less since the older versions of the index format could be
smaller. Request reading up to buf_size, but accept if we only got
the header. */
pos = 0;
do {
ret = pread(index->fd, PTR_OFFSET(buf, pos),
buf_size - pos, pos);
if (ret > 0)
pos += ret;
} while (ret > 0 && pos < sizeof(struct mail_index_header));
*pos_r = pos;
return ret;
}
static int
mail_index_read_map(struct mail_index *index, struct mail_index_map *map,
bool *retry_r, bool try_retry)
{
const struct mail_index_header *hdr;
struct stat st;
unsigned char buf[512];
void *data = NULL;
ssize_t ret;
size_t pos, records_size;
unsigned int records_count;
i_assert(map->mmap_base == NULL);
*retry_r = FALSE;
ret = mail_index_read_header(index, buf, sizeof(buf), &pos);
hdr = (const struct mail_index_header *)buf;
if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
/* major version change - handle silently */
return 0;
}
if (fstat(index->fd, &st) < 0) {
mail_index_set_syscall_error(index, "fstat()");
return -1;
}
if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
(ret > 0 || pos >= hdr->base_header_size)) {
if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
hdr->header_size < hdr->base_header_size) {
mail_index_set_error(index, "Corrupted index file %s: "
"Corrupted header sizes (base %u, full %u)",
index->filepath, hdr->base_header_size,
hdr->header_size);
return 0;
}
if (hdr->header_size > (uoff_t)st.st_size) {
mail_index_set_error(index, "Corrupted index file %s: "
"Corrupted header size (%u > %"PRIuUOFF_T")",
index->filepath, hdr->header_size,
st.st_size);
return 0;
}
if (pos > hdr->header_size)
pos = hdr->header_size;
/* place the base header into memory. */
buffer_reset(map->hdr_copy_buf);
buffer_append(map->hdr_copy_buf, buf, pos);
if (pos != hdr->header_size) {
/* @UNSAFE: read the rest of the header into memory */
data = buffer_append_space_unsafe(map->hdr_copy_buf,
hdr->header_size -
pos);
ret = pread_full(index->fd, data,
hdr->header_size - pos, pos);
}
}
if (ret > 0) {
/* header read, read the records now. */
records_size = (size_t)hdr->messages_count * hdr->record_size;
if ((uoff_t)st.st_size - hdr->header_size < records_size ||
(hdr->record_size != 0 &&
records_size / hdr->record_size != hdr->messages_count)) {
records_count = (st.st_size - hdr->header_size) /
hdr->record_size;
mail_index_set_error(index, "Corrupted index file %s: "
"messages_count too large (%u > %u)",
index->filepath, hdr->messages_count,
records_count);
return 0;
}
if (map->buffer == NULL) {
map->buffer = buffer_create_dynamic(default_pool,
records_size);
}
/* @UNSAFE */
buffer_set_used_size(map->buffer, 0);
data = buffer_append_space_unsafe(map->buffer, records_size);
ret = pread_full(index->fd, data, records_size,
hdr->header_size);
}
if (ret < 0) {
if (errno == ESTALE && try_retry) {
/* a new index file was renamed over this one. */
*retry_r = TRUE;
return 0;
}
mail_index_set_syscall_error(index, "pread_full()");
return -1;
}
if (ret == 0) {
mail_index_set_error(index,
"Corrupted index file %s: File too small",
index->filepath);
return 0;
}
map->records = data;
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
map->hdr_base = map->hdr_copy_buf->data;
index->sync_log_file_seq = hdr->log_file_seq;
index->sync_log_file_offset = hdr->log_file_int_offset;
return 1;
}
bool mail_index_is_ext_synced(struct mail_transaction_log_view *log_view,
struct mail_index_map *map)
{
uint32_t prev_seq;
uoff_t prev_offset;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
return prev_seq < map->hdr.log_file_seq ||
(prev_seq == map->hdr.log_file_seq &&
prev_offset < map->hdr.log_file_ext_offset);
}
static int mail_index_sync_from_transactions(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
const struct mail_index_header *map_hdr = &(*map)->hdr;
struct mail_index_view *view;
struct mail_transaction_log_view *log_view;
struct mail_index_sync_map_ctx sync_map_ctx;
struct mail_index_header hdr;
const struct mail_transaction_header *thdr;
const void *tdata;
uint32_t prev_seq, max_seq;
uoff_t prev_offset, max_offset;
size_t pos;
int ret;
bool skipped, check_ext_offsets, broken;
if (sync_to_index) {
/* read the real log position where we are supposed to be
synced */
ret = mail_index_read_header(index, &hdr, sizeof(hdr), &pos);
if (ret < 0 && errno != ESTALE) {
mail_index_set_syscall_error(index, "pread()");
return -1;
}
if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
return 0;
if (map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset == hdr.log_file_int_offset) {
/* nothing to do */
return 1;
}
if (map_hdr->log_file_seq > hdr.log_file_seq ||
(map_hdr->log_file_seq == hdr.log_file_seq &&
map_hdr->log_file_int_offset > hdr.log_file_int_offset)) {
/* we went too far, have to re-read the file */
return 0;
}
if (map_hdr->log_file_ext_offset !=
map_hdr->log_file_int_offset ||
hdr.log_file_ext_offset != hdr.log_file_int_offset) {
/* too much trouble to get this right. */
return 0;
}
max_seq = hdr.log_file_seq;
max_offset = hdr.log_file_int_offset;
} else {
/* sync everything there is */
max_seq = (uint32_t)-1;
max_offset = (uoff_t)-1;
}
log_view = mail_transaction_log_view_open(index->log);
if (mail_transaction_log_view_set(log_view,
map_hdr->log_file_seq,
map_hdr->log_file_int_offset,
max_seq, max_offset,
MAIL_TRANSACTION_TYPE_MASK) <= 0) {
/* can't use it. sync by re-reading index. */
mail_transaction_log_view_close(&log_view);
return 0;
}
index->map = *map;
view = mail_index_view_open(index);
mail_index_sync_map_init(&sync_map_ctx, view,
MAIL_INDEX_SYNC_HANDLER_HEAD);
check_ext_offsets = TRUE; broken = FALSE;
while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
&skipped)) > 0) {
if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0 &&
check_ext_offsets) {
if (mail_index_is_ext_synced(log_view, index->map))
continue;
check_ext_offsets = FALSE;
}
if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
ret = 0;
broken = TRUE;
break;
}
}
if (ret == 0 && !broken)
ret = 1;
mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
&prev_offset);
i_assert(prev_seq <= max_seq &&
(prev_seq != max_seq || prev_offset <= max_offset));
index->map->hdr.log_file_seq = prev_seq;
index->map->hdr.log_file_int_offset =
index->map->hdr.log_file_ext_offset = prev_offset;
mail_index_sync_map_deinit(&sync_map_ctx);
mail_index_view_close(&view);
mail_transaction_log_view_close(&log_view);
*map = index->map;
index->map = NULL;
if (sync_to_index && ret > 0) {
/* make sure we did everything right. note that although the
message counts should be equal, the flag counters may not */
i_assert(hdr.messages_count == (*map)->hdr.messages_count);
i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq);
i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset);
i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset);
}
return ret;
}
static int mail_index_read_map_with_retry(struct mail_index *index,
struct mail_index_map **map,
bool sync_to_index)
{
mail_index_sync_lost_handler_t *const *handlers;
unsigned int i, count;
int ret;
bool retry;
if (index->log_locked) {
/* we're most likely syncing the index and we really don't
want to read more than what was synced last time. */
sync_to_index = TRUE;
}
if ((*map)->hdr.indexid != 0 && index->log != NULL) {
/* we're not creating the index, or opening transaction log.
sync this as a view from transaction log. */
index->sync_update = TRUE;
ret = mail_index_sync_from_transactions(index, map,
sync_to_index);
index->sync_update = FALSE;
if (ret != 0)
return ret;
/* transaction log lost/broken, fallback to re-reading it */
}
/* notify all "sync lost" handlers */
handlers = array_get(&index->sync_lost_handlers, &count);
for (i = 0; i < count; i++)
(*handlers[i])(index);
for (i = 0;; i++) {
ret = mail_index_read_map(index, *map, &retry,
i < MAIL_INDEX_ESTALE_RETRY_COUNT);
if (ret != 0 || !retry)
return ret;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
}
static int mail_index_map_try_existing(struct mail_index *index)
{
struct mail_index_map *map = index->map;
const struct mail_index_header *hdr;
size_t used_size;
int ret;
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
return 0;
hdr = map->mmap_base;
/* always check corrupted-flag to avoid errors later */
if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
return -1;
used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
if (map->mmap_size >= used_size && map->hdr_base == hdr) {
map->records_count = hdr->messages_count;
mail_index_map_copy_hdr(map, hdr);
/* make sure the header is still valid. it also re-parses
extensions although they shouldn't change without the whole
index being recreated */
ret = mail_index_check_header(index, map);
if (ret > 0)
return 1;
/* broken. fallback to re-mmaping which will catch it */
}
return 0;
}
int mail_index_map(struct mail_index *index, bool force)
{
struct mail_index_map *map;
int ret;
i_assert(!index->mapping);
i_assert(index->map == NULL || index->map->refcount > 0);
i_assert(index->lock_type != F_UNLCK);
if (MAIL_INDEX_IS_IN_MEMORY(index)) {
if (index->map == NULL)
mail_index_create_in_memory(index, NULL);
return 1;
}
index->mapping = TRUE;
if (!force && index->map != NULL) {
i_assert(index->hdr != NULL);
ret = mail_index_map_try_existing(index);
if (ret != 0) {
index->mapping = FALSE;
return ret;
}
if (index->lock_type == F_WRLCK) {
/* we're syncing, don't break the mapping */
index->mapping = FALSE;
return 1;
}
}
if (index->map != NULL && index->map->refcount > 1) {
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
if (!index->mmap_disable) {
map = NULL;
} else {
/* create a copy of the mapping instead so we don't
have to re-read it */
map = mail_index_map_clone(index->map,
index->map->hdr.record_size);
}
index->map->refcount--;
index->map = NULL;
} else {
map = index->map;
}
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
map->hdr_copy_buf =
buffer_create_dynamic(default_pool, sizeof(map->hdr));
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
i_assert(!map->write_to_disk);
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
mail_index_set_syscall_error(index, "munmap()");
map->mmap_base = NULL;
}
index->hdr = NULL;
index->map = NULL;
if (!index->mmap_disable)
ret = mail_index_mmap(index, map);
else
ret = mail_index_read_map_with_retry(index, &map, force);
i_assert(index->map == NULL);
if (ret > 0) {
ret = mail_index_check_header(index, map);
if (ret < 0)
ret = 0;
else if (ret == 0) {
index->fsck = TRUE;
ret = 1;
}
}
if (ret <= 0) {
mail_index_map_clear(index, map);
mail_index_unmap(index, &map);
index->mapping = FALSE;
return ret;
}
index->hdr = &map->hdr;
index->map = map;
i_assert(map->hdr.messages_count == map->records_count);
index->mapping = FALSE;
return 1;
}
int mail_index_get_latest_header(struct mail_index *index,
struct mail_index_header *hdr_r)
{
size_t pos;
unsigned int i;
int ret;
if (!index->mmap_disable) {
ret = mail_index_map(index, FALSE);
if (ret > 0)
*hdr_r = *index->hdr;
else
memset(hdr_r, 0, sizeof(*hdr_r));
return ret;
}
for (i = 0;; i++) {
ret = mail_index_read_header(index, hdr_r, sizeof(*hdr_r),
&pos);
if (ret <= 0 || errno != ESTALE ||
i == MAIL_INDEX_ESTALE_RETRY_COUNT)
break;
/* ESTALE - reopen index file */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
ret = mail_index_try_open_only(index);
if (ret <= 0) {
if (ret == 0) {
/* the file was lost */
errno = ENOENT;
mail_index_set_syscall_error(index, "open()");
}
return -1;
}
}
if (ret < 0)
mail_index_set_syscall_error(index, "pread_full()");
return ret;
}
struct mail_index_map *
mail_index_map_clone(const struct mail_index_map *map, uint32_t new_record_size)
{
struct mail_index_map *mem_map;
struct mail_index_header *hdr;
struct mail_index_ext *extensions;
void *src, *dest;
size_t size, copy_size;
unsigned int i, count;
size = map->records_count * new_record_size;
mem_map = i_new(struct mail_index_map, 1);
mem_map->refcount = 1;
mem_map->buffer = buffer_create_dynamic(default_pool, size);
if (map->hdr.record_size == new_record_size)
buffer_append(mem_map->buffer, map->records, size);
else {
copy_size = I_MIN(map->hdr.record_size, new_record_size);
src = map->records;
for (i = 0; i < map->records_count; i++) {
dest = buffer_append_space_unsafe(mem_map->buffer,
new_record_size);
memcpy(dest, src, copy_size);
src = PTR_OFFSET(src, map->hdr.record_size);
}
}
mem_map->records = buffer_get_modifiable_data(mem_map->buffer, NULL);
mem_map->records_count = map->records_count;
mem_map->hdr_copy_buf =
buffer_create_dynamic(default_pool, map->hdr.header_size);
if (map->hdr.base_header_size < sizeof(*hdr))
buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr));
buffer_write(mem_map->hdr_copy_buf, 0,
&map->hdr, map->hdr.base_header_size);
buffer_append(mem_map->hdr_copy_buf,
CONST_PTR_OFFSET(map->hdr_base,
map->hdr.base_header_size),
map->hdr.header_size - map->hdr.base_header_size);
hdr = buffer_get_modifiable_data(mem_map->hdr_copy_buf, NULL);
if (hdr->base_header_size < sizeof(*hdr))
hdr->base_header_size = sizeof(*hdr);
hdr->record_size = new_record_size;
mem_map->hdr = *hdr;
mem_map->hdr_base = hdr;
/* if we're syncing transaction log into memory and later use the
mapping for updating the index, we need to remember what has
changed */
mem_map->write_atomic = map->write_atomic;
if (map->write_to_disk) {
mem_map->write_seq_first = map->write_seq_first;
mem_map->write_seq_last = map->write_seq_last;
}
/* copy extensions */
if (array_is_created(&map->ext_id_map)) {
count = array_count(&map->ext_id_map);
mail_index_map_init_extbufs(mem_map, count + 2);
array_append_array(&mem_map->extensions, &map->extensions);
array_append_array(&mem_map->ext_id_map, &map->ext_id_map);
/* fix the name pointers to use our own pool */
extensions = array_get_modifiable(&mem_map->extensions, &count);
for (i = 0; i < count; i++) {
i_assert(extensions[i].record_offset +
extensions[i].record_size <= hdr->record_size);
extensions[i].name = p_strdup(mem_map->extension_pool,
extensions[i].name);
}
}
return mem_map;
}
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id;
if (!array_is_created(&map->ext_id_map) ||
ext_id >= array_count(&map->ext_id_map))
return 0;
id = array_idx(&map->ext_id_map, ext_id);
*idx_r = *id;
return *idx_r != (uint32_t)-1;
}
static int mail_index_try_open_only(struct mail_index *index)
{
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
/* Note that our caller must close index->fd by itself.
mail_index_reopen() for example wants to revert back to old
index file if opening the new one fails. */
index->fd = nfs_safe_open(index->filepath, O_RDWR);
index->readonly = FALSE;
if (index->fd == -1 && errno == EACCES) {
index->fd = open(index->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (index->fd == -1) {
if (errno != ENOENT)
return mail_index_set_syscall_error(index, "open()");
/* have to create it */
return 0;
}
return 1;
}
static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
i_assert(index->fd == -1);
i_assert(index->lock_type == F_UNLCK);
if (lock_id_r != NULL)
*lock_id_r = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
ret = mail_index_try_open_only(index);
if (ret <= 0)
return ret;
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
(void)close(index->fd);
index->fd = -1;
return -1;
}
ret = mail_index_map(index, FALSE);
if (ret == 0) {
/* it's corrupted - recreate it */
mail_index_unlock(index, lock_id);
if (lock_id_r != NULL)
*lock_id_r = 0;
i_assert(index->file_lock == NULL);
(void)close(index->fd);
index->fd = -1;
} else {
if (lock_id_r != NULL)
*lock_id_r = lock_id;
else
mail_index_unlock(index, lock_id);
}
return ret;
}
int mail_index_write_base_header(struct mail_index *index,
const struct mail_index_header *hdr)
{
size_t hdr_size;
hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
memcpy(index->map->mmap_base, hdr, hdr_size);
if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
index->map->hdr = *hdr;
} else {
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
mail_index_set_syscall_error(index,
"pwrite_full()");
return -1;
}
}
index->map->hdr = *hdr;
buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
}
return 0;
}
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
mode_t old_mask;
const char *path;
int fd;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
old_mask = umask(0);
fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
umask(old_mask);
if (fd == -1)
return mail_index_file_set_syscall_error(index, path, "open()");
if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, path, "fchown()");
return -1;
}
return fd;
}
static int mail_index_create(struct mail_index *index,
struct mail_index_header *hdr)
{
const char *path;
uint32_t seq;
uoff_t offset;
int ret;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->lock_type == F_UNLCK);
/* log file lock protects index creation */
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
ret = mail_index_try_open(index, NULL);
if (ret != 0) {
mail_transaction_log_sync_unlock(index->log);
return ret < 0 ? -1 : 0;
}
/* mark the existing log file as synced */
hdr->log_file_seq = seq;
hdr->log_file_int_offset = offset;
hdr->log_file_ext_offset = offset;
/* create it fully in index.tmp first */
index->fd = mail_index_create_tmp_file(index, &path);
if (index->fd == -1)
ret = -1;
else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
ret = -1;
} else {
index->lock_type = F_WRLCK;
ret = mail_index_map(index, FALSE);
index->lock_type = F_UNLCK;
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
mail_index_set_error(index,
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
if (unlink(path) < 0 && errno != ENOENT) {
mail_index_file_set_syscall_error(index, path,
"unlink()");
}
} else {
/* make it visible to others */
if (rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
}
mail_transaction_log_sync_unlock(index->log);
return ret;
}
static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
#ifndef WORDS_BIGENDIAN
hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->indexid = now;
hdr->next_uid = 1;
}
static void mail_index_create_in_memory(struct mail_index *index,
const struct mail_index_header *hdr)
{
struct mail_index_header tmp_hdr;
struct mail_index_map tmp_map;
if (hdr == NULL) {
mail_index_header_init(&tmp_hdr);
hdr = &tmp_hdr;
}
memset(&tmp_map, 0, sizeof(tmp_map));
tmp_map.hdr = *hdr;
tmp_map.hdr_base = hdr;
/* a bit kludgy way to do this, but it initializes everything
nicely and correctly */
index->map = mail_index_map_clone(&tmp_map, hdr->record_size);
index->hdr = &index->map->hdr;
}
/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
bool created = FALSE;
ret = mail_index_try_open(index, &lock_id);
if (ret > 0)
hdr = *index->hdr;
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 &&
!MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
index->indexid = hdr.indexid;
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL) {
if (ret == 0)
index->hdr = NULL;
return -1;
}
if (index->fd == -1) {
mail_index_header_init(&hdr);
index->hdr = &hdr;
/* index->indexid may be updated by transaction log opening,
in case someone else had already created a new log file */
hdr.indexid = index->indexid;
if (lock_id != 0) {
mail_index_unlock(index, lock_id);
lock_id = 0;
}
if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
if (mail_index_create(index, &hdr) < 0) {
/* fallback to in-memory index */
mail_index_move_to_memory(index);
mail_index_create_in_memory(index, &hdr);
}
} else {
mail_index_create_in_memory(index, &hdr);
}
created = TRUE;
}
i_assert(index->hdr != &hdr);
if (lock_id == 0) {
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
return -1;
}
index->cache = created ? mail_cache_create(index) :
mail_cache_open_or_create(index);
mail_index_unlock(index, lock_id);
return 1;
}
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags,
enum file_lock_method lock_method)
{
int i = 0, ret;
if (index->opened) {
if (index->hdr != NULL &&
(index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
/* corrupted, reopen files */
mail_index_close(index);
} else {
return 1;
}
}
index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ?
i_strdup("(in-memory index)") :
i_strconcat(index->dir, "/", index->prefix, NULL);
for (;;) {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id = 2;
index->readonly = FALSE;
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
index->log_locked = FALSE;
index->mmap_disable =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
index->mmap_no_write =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
index->use_excl_dotlocks =
(flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0;
index->fsync_disable =
(flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0;
index->lock_method = lock_method;
/* don't even bother to handle dotlocking without mmap being
disabled. that combination simply doesn't make any sense */
if (lock_method == FILE_LOCK_METHOD_DOTLOCK &&
!index->mmap_disable) {
i_fatal("lock_method=dotlock and mmap_disable=no "
"combination isn't supported. "
"You don't _really_ want it anyway.");
}
ret = mail_index_open_files(index, flags);
if (ret <= 0)
break;
index->opened = TRUE;
if (index->fsck) {
index->fsck = FALSE;
ret = mail_index_fsck(index);
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
}
if (ret <= 0)
mail_index_close(index);
return ret;
}
void mail_index_close(struct mail_index *index)
{
if (index->log != NULL)
mail_transaction_log_close(&index->log);
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->cache != NULL)
mail_cache_free(&index->cache);
if (index->file_lock != NULL)
file_lock_free(&index->file_lock);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
}
i_free_and_null(index->copy_lock_path);
i_free_and_null(index->filepath);
index->indexid = 0;
index->opened = FALSE;
}
int mail_index_reopen(struct mail_index *index, int fd)
{
struct mail_index_map *old_map;
struct file_lock *old_file_lock;
unsigned int old_shared_locks, old_lock_id, lock_id = 0;
int ret, old_fd, old_lock_type;
i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
i_assert(index->excl_lock_count == 0);
old_map = index->map;
old_fd = index->fd;
old_map->refcount++;
/* new file, new locks. the old fd can keep its locks, they don't
matter anymore as no-one's going to modify the file. */
old_lock_type = index->lock_type;
old_lock_id = index->lock_id;
old_shared_locks = index->shared_lock_count;
old_file_lock = index->file_lock;
if (index->lock_type == F_RDLCK)
index->lock_type = F_UNLCK;
index->lock_id += 2;
index->shared_lock_count = 0;
index->file_lock = NULL;
if (fd != -1) {
index->fd = fd;
ret = 0;
} else {
ret = mail_index_try_open_only(index);
if (ret > 0)
ret = mail_index_lock_shared(index, FALSE, &lock_id);
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
/* read the new mapping. note that with mmap_disable we want
to keep the old mapping in index->map so we can update it
by reading transaction log. */
if (mail_index_map(index, TRUE) <= 0)
ret = -1;
}
if (lock_id != 0)
mail_index_unlock(index, lock_id);
if (ret == 0) {
mail_index_unmap(index, &old_map);
if (old_file_lock != NULL)
file_lock_free(&old_file_lock);
if (close(old_fd) < 0)
mail_index_set_syscall_error(index, "close()");
} else {
if (index->map != NULL)
mail_index_unmap(index, &index->map);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
}
index->map = old_map;
index->hdr = &index->map->hdr;
index->fd = old_fd;
index->file_lock = old_file_lock;
index->lock_type = old_lock_type;
index->lock_id = old_lock_id;
index->shared_lock_count = old_shared_locks;
}
return ret;
}
int mail_index_reopen_if_needed(struct mail_index *index)
{
struct stat st1, st2;
i_assert(index->copy_lock_path == NULL);
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (fstat(index->fd, &st1) < 0) {
if (errno == ESTALE) {
/* deleted, reopen */
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
}
return mail_index_set_syscall_error(index, "fstat()");
}
if (nfs_safe_stat(index->filepath, &st2) < 0) {
mail_index_set_syscall_error(index, "stat()");
if (errno != ENOENT)
return -1;
/* lost it? recreate later */
mail_index_mark_corrupted(index);
return -1;
}
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
int mail_index_refresh(struct mail_index *index)
{
unsigned int lock_id;
int ret;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
if (index->excl_lock_count > 0) {
/* we have index exclusively locked, nothing could
have changed. */
return 0;
}
if (!index->mmap_disable) {
/* reopening is all we need */
return mail_index_reopen_if_needed(index);
}
i_assert(!index->mapping);
/* mail_index_map() simply reads latest changes from transaction log,
which makes us fully refreshed. */
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
ret = mail_index_map(index, FALSE);
mail_index_unlock(index, lock_id);
return ret <= 0 ? -1 : 0;
}
struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
return index->cache;
}
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
va_list va;
i_free(index->error);
if (fmt == NULL)
index->error = NULL;
else {
va_start(va, fmt);
index->error = i_strdup_vprintf(fmt, va);
va_end(va);
i_error("%s", index->error);
}
return -1;
}
void mail_index_set_inconsistent(struct mail_index *index)
{
index->indexid = 0;
}
int mail_index_move_to_memory(struct mail_index *index)
{
struct mail_index_map *map;
int ret = 0;
if (MAIL_INDEX_IS_IN_MEMORY(index))
return 0;
/* set the index as being into memory */
i_free_and_null(index->dir);
if (index->map == NULL) {
/* mbox file was never even opened. just mark it as being in
memory and let the caller re-open the index. */
i_assert(index->fd == -1);
return -1;
}
/* move index map to memory */
map = mail_index_map_clone(index->map, index->map->hdr.record_size);
mail_index_unmap(index, &index->map);
index->map = map;
index->hdr = &map->hdr;
if (index->log != NULL) {
/* move transaction log to memory */
if (mail_transaction_log_move_to_memory(index->log) < 0)
ret = -1;
}
if (index->file_lock != NULL)
file_lock_free(&index->file_lock);
/* close the index file. */
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
return ret;
}
void mail_index_mark_corrupted(struct mail_index *index)
{
struct mail_index_header hdr;
mail_index_set_inconsistent(index);
if (index->readonly || index->map == NULL)
return;
hdr = *index->hdr;
hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
if (mail_index_write_base_header(index, &hdr) == 0) {
if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0)
mail_index_set_syscall_error(index, "fsync()");
}
}
int mail_index_set_syscall_error(struct mail_index *index,
const char *function)
{
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with index file %s: %m",
function, index->filepath);
}
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function)
{
i_assert(filepath != NULL);
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with file %s: %m",
function, filepath);
}
enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
if (index->error != NULL)
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
const char *mail_index_get_error_message(struct mail_index *index)
{
return index->error;
}
void mail_index_reset_error(struct mail_index *index)
{
if (index->error != NULL) {
i_free(index->error);
index->error = NULL;
}
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
}
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
unsigned char buf[4];
i_assert(offset < 0x40000000);
i_assert((offset & 3) == 0);
offset >>= 2;
buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
buf[3] = 0x80 | (offset & 0x0000007f);
return *((uint32_t *) buf);
}
uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
const unsigned char *buf = (const unsigned char *) &offset;
if ((offset & 0x80808080) != 0x80808080)
return 0;
return (((uint32_t)buf[3] & 0x7f) << 2) |
(((uint32_t)buf[2] & 0x7f) << 9) |
(((uint32_t)buf[1] & 0x7f) << 16) |
(((uint32_t)buf[0] & 0x7f) << 23);
}