mail-index.c revision 7797aa2479e99aeb71057b7a2584b2cb72e4d3f8
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "lib.h"
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include "buffer.h"
d9076f5939edf5d20a261494b1a861dcbb0d32e2Timo Sirainen#include "file-lock.h"
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen#include "mmap-util.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen#include "read-full.h"
5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen#include "write-full.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "mail-index-private.h"
146f9076cd456ea1e9b3f8536456d9d3c962fadbStephan Bosch#include "mail-transaction-log.h"
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen#include "mail-cache.h"
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
8c909e451d14075c05d90382cf8eebc4e354f569Timo Sirainen#include <stdio.h>
8c909e451d14075c05d90382cf8eebc4e354f569Timo Sirainen#include <stddef.h>
8c909e451d14075c05d90382cf8eebc4e354f569Timo Sirainen#include <time.h>
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen#include <sys/stat.h>
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainen
573f0491a5733fe21fa062a455acb4790b4e0499Timo Sirainenstatic int mail_index_try_open_only(struct mail_index *index);
3ed2d0f6b5e67e2663d44489d9da3176823789a8Timo Sirainen
65f8fb656051f1059f7b5a2da9c5555adcc30439Timo Sirainenstruct mail_index *mail_index_alloc(const char *dir, const char *prefix)
1a5573ebc32fae2fe576ec544e1781323c1db609Timo Sirainen{
1a5573ebc32fae2fe576ec544e1781323c1db609Timo Sirainen struct mail_index *index;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index = i_new(struct mail_index, 1);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->dir = i_strdup(dir);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->prefix = i_strdup(prefix);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->fd = -1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->extension_pool = pool_alloconly_create("extension", 256);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->extensions =
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen buffer_create_dynamic(index->extension_pool, 64, (size_t)-1);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen index->mode = 0600;
f7539a17ea306191b53b8f5e752e228937df9ec3Timo Sirainen index->gid = (gid_t)-1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return index;
2dd39e478269d6fb0bb26d12b394aa30ee965e38Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainenvoid mail_index_free(struct mail_index *index)
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen{
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen mail_index_close(index);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen pool_unref(index->extension_pool);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_free(index->error);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_free(index->dir);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen i_free(index->prefix);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen i_free(index);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen}
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainenvoid mail_index_set_permissions(struct mail_index *index,
3e564425db51f3921ce4de11859777135fdedd15Timo Sirainen mode_t mode, gid_t gid)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen{
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen index->mode = mode & 0666;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->gid = gid;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen}
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenuint32_t mail_index_ext_register(struct mail_index *index, const char *name,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen uint32_t hdr_size, uint16_t record_size)
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen{
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen const struct mail_index_ext *extensions;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen struct mail_index_ext ext;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen size_t ext_count;
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen unsigned int i;
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen extensions = buffer_get_data(index->extensions, &ext_count);
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen ext_count /= sizeof(*extensions);
dfaefeabae939803ceb8c503101e86b5496541d1Timo Sirainen
dfaefeabae939803ceb8c503101e86b5496541d1Timo Sirainen /* see if it's there already */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen for (i = 0; i < ext_count; i++) {
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen if (strcmp(extensions[i].name, name) == 0) {
8b31f966d9688e07672ef1958dcbdb7686523c04Timo Sirainen i_assert(extensions[i].hdr_size == hdr_size);
8b31f966d9688e07672ef1958dcbdb7686523c04Timo Sirainen i_assert(extensions[i].record_size == record_size);
9847ec56efa15fa063eea9988eee2d4ed9ec7d58Timo Sirainen return i;
9847ec56efa15fa063eea9988eee2d4ed9ec7d58Timo Sirainen }
9847ec56efa15fa063eea9988eee2d4ed9ec7d58Timo Sirainen }
d46a1e3f999dda802dc5137e883adcd7a6629cd3Timo Sirainen
d46a1e3f999dda802dc5137e883adcd7a6629cd3Timo Sirainen i_assert(hdr_size % 4 == 0);
d46a1e3f999dda802dc5137e883adcd7a6629cd3Timo Sirainen i_assert(record_size % 4 == 0);
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen memset(&ext, 0, sizeof(ext));
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen ext.name = p_strdup(index->extension_pool, name);
d1e843e77f4760e303c53d9fce10123fc8d230a1Timo Sirainen ext.hdr_size = hdr_size;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen ext.record_size = record_size;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen buffer_append(index->extensions, &ext, sizeof(ext));
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen return ext_count;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen}
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainenstatic void mail_index_map_init_extbufs(struct mail_index_map *map,
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen unsigned int initial_count)
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen{
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen size_t ext_size, ext_id_map_size, size;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ext_size = initial_count * sizeof(struct mail_index_ext);
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ext_id_map_size = initial_count * sizeof(uint32_t);
97ae33602db7d5bc8eede82512a965d49ab8853bTimo Sirainen if (map->extension_pool == NULL) {
97ae33602db7d5bc8eede82512a965d49ab8853bTimo Sirainen size = ext_size + ext_id_map_size +
97ae33602db7d5bc8eede82512a965d49ab8853bTimo Sirainen (initial_count * 20); /* for names */
97ae33602db7d5bc8eede82512a965d49ab8853bTimo Sirainen map->extension_pool =
97ae33602db7d5bc8eede82512a965d49ab8853bTimo Sirainen pool_alloconly_create("extensions",
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen nearest_power(size));
57a8c6a95e4bce3eeaba36985adb81c07dd683ffTimo Sirainen } else {
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen p_clear(map->extension_pool);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen }
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->extensions = buffer_create_dynamic(map->extension_pool,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext_size, (size_t)-1);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen map->ext_id_map = buffer_create_dynamic(map->extension_pool,
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen ext_id_map_size, (size_t)-1);
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen}
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainenuint32_t mail_index_map_register_ext(struct mail_index *index,
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen struct mail_index_map *map,
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen const char *name, uint32_t hdr_offset,
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen uint32_t hdr_size, uint32_t record_size)
eecb235c14b49c01774134ea593c266f2d2c2be1Timo Sirainen{
eecb235c14b49c01774134ea593c266f2d2c2be1Timo Sirainen const struct mail_index_ext *last_ext;
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen struct mail_index_ext *ext;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen size_t size;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen uint32_t idx, ext_id;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
97ae33602db7d5bc8eede82512a965d49ab8853bTimo Sirainen if (map->extensions == NULL) {
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen mail_index_map_init_extbufs(map, 5);
f1743785713e7632459d623d5df2108f4b93accbTimo Sirainen last_ext = NULL;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen idx = 0;
70ead6466f9baa8294e71fc2fba0a4f54f488b5eTimo Sirainen } else {
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen last_ext = buffer_get_data(map->extensions, &size);
ccc895c0358108d2304239063e940b7d75f364abTimo Sirainen idx = size / sizeof(*last_ext);
8d630c15a8ed6f85553467c3a231a273defca5f6Timo Sirainen if (idx == 0)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen last_ext = NULL;
ee116df08d0fdab703483e18fe8076b2ef9fd9d7Timo Sirainen else
c5ab90cfad9cc3e33bcb1baeb30ffc82a7b7053aTimo Sirainen last_ext += idx - 1;
c5ab90cfad9cc3e33bcb1baeb30ffc82a7b7053aTimo Sirainen }
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen
9fd2181788a61500641c66aec0f8c746b19bf830Timo Sirainen ext = buffer_append_space_unsafe(map->extensions, sizeof(*ext));
9fd2181788a61500641c66aec0f8c746b19bf830Timo Sirainen memset(ext, 0, sizeof(*ext));
9fd2181788a61500641c66aec0f8c746b19bf830Timo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ext->name = p_strdup(map->extension_pool, name);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen ext->hdr_offset = hdr_offset;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen ext->hdr_size = hdr_size;
c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainen ext->record_size = record_size;
9fd2181788a61500641c66aec0f8c746b19bf830Timo Sirainen
9fd2181788a61500641c66aec0f8c746b19bf830Timo Sirainen if (last_ext != NULL) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen ext->record_offset = last_ext->record_offset +
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen last_ext->record_size;
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen } else {
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen ext->record_offset = sizeof(struct mail_index_record);
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen }
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen ext_id = mail_index_ext_register(index, name, hdr_size, record_size);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen buffer_write(map->ext_id_map, ext_id * sizeof(uint32_t),
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen &idx, sizeof(idx));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return idx;
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen}
cff1f182205e674285cf3ff446a0dcf7afea277dTimo Sirainen
cff1f182205e674285cf3ff446a0dcf7afea277dTimo Sirainenstatic int mail_index_read_extensions(struct mail_index *index,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen struct mail_index_map *map)
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen{
e03d986a74128f5ba30fcfda9f6e36578f5d8decTimo Sirainen const struct mail_index_ext_header *ext_hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen unsigned int i, old_count;
f318b3dbe2acc177b8ee1c160e4b5b14e7f2cd41Timo Sirainen const char *name;
f318b3dbe2acc177b8ee1c160e4b5b14e7f2cd41Timo Sirainen uint32_t ext_id, offset, name_offset;
f318b3dbe2acc177b8ee1c160e4b5b14e7f2cd41Timo Sirainen
f318b3dbe2acc177b8ee1c160e4b5b14e7f2cd41Timo Sirainen offset = map->hdr->base_header_size;
f318b3dbe2acc177b8ee1c160e4b5b14e7f2cd41Timo Sirainen if (offset == map->hdr->header_size && map->extension_pool == NULL) {
f318b3dbe2acc177b8ee1c160e4b5b14e7f2cd41Timo Sirainen /* nothing to do, skip allocatations and all */
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen return 1;
d3280fe317a4598c0868cc440e7a1191c06d0db3Timo Sirainen }
d3280fe317a4598c0868cc440e7a1191c06d0db3Timo Sirainen
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen old_count = index->extensions->used / sizeof(struct mail_index_ext);
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mail_index_map_init_extbufs(map, old_count + 5);
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen
6469cf211a57433335641725dc236ebb2b9fdd3bTimo Sirainen ext_id = (uint32_t)-1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen for (i = 0; i < old_count; i++)
50c617761ee9653bd44646a95178773a3686d62eTimo Sirainen buffer_append(map->ext_id_map, &ext_id, sizeof(ext_id));
6cc4cce2078aca138fbce19305e69e77edcee614Timo Sirainen
6cc4cce2078aca138fbce19305e69e77edcee614Timo Sirainen name = map->hdr_base;
50c617761ee9653bd44646a95178773a3686d62eTimo Sirainen while (offset < map->hdr->header_size) {
50c617761ee9653bd44646a95178773a3686d62eTimo Sirainen name_offset = offset;
50c617761ee9653bd44646a95178773a3686d62eTimo Sirainen
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen while (offset < map->hdr->header_size && name[offset] != '\0')
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen offset++;
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen if (offset == map->hdr->header_size) {
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen "Header extension name doesn't end with NUL",
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen index->filepath);
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen return -1;
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen }
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen offset++;
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen while (offset < map->hdr->header_size && (offset % 4) != 0)
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen offset++;
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen if (offset + sizeof(*ext_hdr) > map->hdr->header_size ||
c0d069950af1dbc6a4e5c3de3bf2e437796e3ae0Timo Sirainen offset + sizeof(*ext_hdr) + ext_hdr->hdr_size >
d3442384ca53d4b18a493db7dd0b000f470419cfTimo Sirainen map->hdr->header_size) {
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen "Header extension goes outside header",
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen index->filepath);
c979eeda1f46483d9c963e265786b701d7683d77Timo Sirainen return -1;
c979eeda1f46483d9c963e265786b701d7683d77Timo Sirainen }
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen
2584e86cc2d8c31ba30a4109cf4ba09d1e37e28aTimo Sirainen mail_index_map_register_ext(index, map, name + name_offset,
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen offset + sizeof(*ext_hdr),
4b41116563110d00330896a568eff1078c382827Timo Sirainen ext_hdr->hdr_size,
4b41116563110d00330896a568eff1078c382827Timo Sirainen ext_hdr->record_size);
4b41116563110d00330896a568eff1078c382827Timo Sirainen
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen offset += sizeof(*ext_hdr) + ext_hdr->hdr_size;
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen }
5137d2d80255938a0f5fb8f3c1a21b34cf11ada3Timo Sirainen return 1;
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mail_index_check_header(struct mail_index *index,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen struct mail_index_map *map)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen{
b2c1349cf07410aefab0f5b17153af9e5cfcf48fTimo Sirainen const struct mail_index_header *hdr = map->hdr;
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen unsigned char compat_data[sizeof(hdr->compat_data)];
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dbe64f3893616a4005c8946be75d2dc8f6823d72Timo Sirainen memset(compat_data, 0, sizeof(compat_data));
8a13b020f90e080570658b18c042e7e352c8b14fTimo Sirainen#ifndef WORDS_BIGENDIAN
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen#endif
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen compat_data[1] = sizeof(uoff_t);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen compat_data[2] = sizeof(time_t);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen /* major version change - handle silently(?) */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return -1;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen }
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen if (memcmp(hdr->compat_data, compat_data, sizeof(compat_data)) != 0) {
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen /* architecture change - handle silently(?) */
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
f3bb2fbe87425dc89a839908985af496f7f65702Timo Sirainen
eef4ba0cc3e78f8c26804c1c9251a76580a41f0cTimo Sirainen if ((map->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
a3ee5ce6ecc8e228ee69300fdd562d7ac8be89a7Timo Sirainen /* we've already complained about it */
bd1b2615928a1e8be190cb0405754f0aec8cac2fTimo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen /* following some extra checks that only take a bit of CPU */
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
2ebeb22b9a8a8bb7fbe2f2e2908478a220792b87Timo Sirainen "uid_validity = 0, next_uid = %u",
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen index->filepath, hdr->next_uid);
bd4d0a1a7c0626452b8d82f37e3ec07267ac9896Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen if (hdr->keywords_mask_size != sizeof(keywords_mask_t)) {
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen "keywords_mask_size mismatch: %d != %d",
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen index->filepath, hdr->keywords_mask_size,
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen (int)sizeof(keywords_mask_t));
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen return -1;
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen if (hdr->record_size < sizeof(struct mail_index_record)) {
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen "record_size too small: %u < %"PRIuSIZE_T,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->filepath, hdr->record_size,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen sizeof(struct mail_index_record));
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return -1;
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen }
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen if (hdr->next_uid == 0)
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen return 0;
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen if (hdr->recent_messages_count > hdr->messages_count ||
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen hdr->seen_messages_count > hdr->messages_count ||
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen hdr->deleted_messages_count > hdr->messages_count)
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen return 0;
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen if (hdr->first_recent_uid_lowwater > hdr->next_uid ||
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen hdr->first_unseen_uid_lowwater > hdr->next_uid ||
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen hdr->first_deleted_uid_lowwater > hdr->next_uid)
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return 0;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return mail_index_read_extensions(index, map);
d938e9e4ec4c0f326dffd5ebe42c1ad893ce7e52Timo Sirainen}
eecb235c14b49c01774134ea593c266f2d2c2be1Timo Sirainen
eecb235c14b49c01774134ea593c266f2d2c2be1Timo Sirainenstatic void mail_index_map_clear(struct mail_index *index,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen struct mail_index_map *map)
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen{
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen if (map->buffer != NULL) {
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen i_assert(map->mmap_base == NULL);
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen buffer_free(map->buffer);
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen map->buffer = NULL;
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen } else if (map->mmap_base != NULL) {
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen i_assert(map->buffer == NULL);
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen if (munmap(map->mmap_base, map->mmap_size) < 0)
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen mail_index_set_syscall_error(index, "munmap()");
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen map->mmap_base = NULL;
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen }
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen if (map->refcount > 0) {
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen map->mmap_size = 0;
546335814920fb6b5b44c68c7803e654eefeae9dTimo Sirainen map->mmap_used_size = 0;
546335814920fb6b5b44c68c7803e654eefeae9dTimo Sirainen map->hdr = NULL;
546335814920fb6b5b44c68c7803e654eefeae9dTimo Sirainen map->records = NULL;
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen map->records_count = 0;
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen }
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen}
eb1572d7c44ebc7b0b039d085c3dbab2ef7043ddTimo Sirainen
eb1572d7c44ebc7b0b039d085c3dbab2ef7043ddTimo Sirainenvoid mail_index_unmap(struct mail_index *index, struct mail_index_map *map)
ca98d6a1bbe73499da758a36bfab2963375c8d06Timo Sirainen{
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen if (--map->refcount > 0)
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen return;
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen i_assert(map->refcount == 0);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen mail_index_map_clear(index, map);
df00412606a00714a6e85383fa87fbdc7cc1fb5bTimo Sirainen if (map->extension_pool != NULL)
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen pool_unref(map->extension_pool);
42507d758b053bb483de58fba55c73a9eb5d3fbaTimo Sirainen buffer_free(map->hdr_copy_buf);
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen i_free(map);
0177594fa5217b02001f4ec8752154fd2b05c545Timo Sirainen}
42507d758b053bb483de58fba55c73a9eb5d3fbaTimo Sirainen
6ae329de09afb7214c906d762320847e05469d53Timo Sirainenstatic void mail_index_unmap_forced(struct mail_index *index,
6ae329de09afb7214c906d762320847e05469d53Timo Sirainen struct mail_index_map *map)
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen{
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen mail_index_map_clear(index, map);
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen mail_index_unmap(index, map);
f81f4bc282cd1944cec187bae89c0701a416ed2aTimo Sirainen}
fcfe85637e1ee14a9fc39c41fd6ceca106301542Timo Sirainen
fcfe85637e1ee14a9fc39c41fd6ceca106301542Timo Sirainenstatic int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
fcfe85637e1ee14a9fc39c41fd6ceca106301542Timo Sirainen{
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen const struct mail_index_header *hdr;
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen struct mail_index_header *mhdr;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen unsigned int records_count;
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen i_assert(!map->write_to_disk);
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen if (map->buffer != NULL) {
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen /* we had temporarily used a buffer, eg. for updating index */
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen buffer_free(map->buffer);
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen map->buffer = NULL;
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen }
7baab0b0b60df7ce9093d0881cd322dff1e79491Timo Sirainen
7baab0b0b60df7ce9093d0881cd322dff1e79491Timo Sirainen map->mmap_base = index->lock_type != F_WRLCK ?
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen mmap_ro_file(index->fd, &map->mmap_size) :
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen mmap_rw_file(index->fd, &map->mmap_size);
f90cbe597c41d5cc91debd371f8648bd8e6ffbc2Timo Sirainen if (map->mmap_base == MAP_FAILED) {
f90cbe597c41d5cc91debd371f8648bd8e6ffbc2Timo Sirainen map->mmap_base = NULL;
33b469d1ca66dd2cc496d2d990b8b98e72952a29Timo Sirainen mail_index_set_syscall_error(index, "mmap()");
33b469d1ca66dd2cc496d2d990b8b98e72952a29Timo Sirainen return -1;
58ba0fe5a6904d3a65cfe268411f4cbb881234eeTimo Sirainen }
58ba0fe5a6904d3a65cfe268411f4cbb881234eeTimo Sirainen
58ba0fe5a6904d3a65cfe268411f4cbb881234eeTimo Sirainen if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
6b0d8106ae51ffc6ce45636b34d2e21cbefca7fdTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
6b0d8106ae51ffc6ce45636b34d2e21cbefca7fdTimo Sirainen "File too small (%"PRIuSIZE_T")",
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen index->filepath, map->mmap_size);
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen return 0;
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen }
eb64c3586d854cddd693f0b811d897399076a441Timo Sirainen
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen hdr = map->mmap_base;
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen map->hdr = hdr;
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen map->mmap_used_size = hdr->header_size +
7f735cb86b2d8abd8f230089065eacfc24e9e5d6Timo Sirainen hdr->messages_count * hdr->record_size;
b3484b5b1f47e4cf112f0e371478a2d7794b31bbTimo Sirainen
b3484b5b1f47e4cf112f0e371478a2d7794b31bbTimo Sirainen if (map->mmap_used_size > map->mmap_size) {
d81131d3bbb4f0befb62a661d1785cf8c84a17e2Timo Sirainen records_count = (map->mmap_size - hdr->header_size) /
d81131d3bbb4f0befb62a661d1785cf8c84a17e2Timo Sirainen hdr->record_size;
9456a4a3e74929f9d3d5b00b93be6d8eb69bc52aTimo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
9456a4a3e74929f9d3d5b00b93be6d8eb69bc52aTimo Sirainen "messages_count too large (%u > %u)",
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen index->filepath, hdr->messages_count,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen records_count);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen return 0;
0d86aa0d47f7393c669c084b34c0537b193688adTimo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
7631f16156aca373004953fe6b01a7f343fb47e0Timo Sirainen if (map->hdr->base_header_size < sizeof(*map->hdr)) {
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen /* header smaller than ours, make a copy so our newer headers
aa247243412a49f9bdebf7255e131dc6ece4ed46Timo Sirainen won't have garbage in them */
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen buffer_reset(map->hdr_copy_buf);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen buffer_append(map->hdr_copy_buf,
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen map->hdr, map->hdr->base_header_size);
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen buffer_append_zero(map->hdr_copy_buf, sizeof(*map->hdr) -
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen map->hdr->base_header_size);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mhdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mhdr->base_header_size = sizeof(*map->hdr);
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen mhdr->header_size = map->hdr_copy_buf->used;
a757f31393b9d6fc7760a9dec8363404ab3ae576Timo Sirainen
a2f250a332dfc1e6cd4ffd196c621eb9dbf7b8a1Timo Sirainen map->hdr = mhdr;
f46885a5b78b15a8d2419f6e5d13b643bd85e41fTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen map->hdr_base = map->mmap_base;
c0225f7f6b43d34dc58c17d3304f0fd60ab89894Timo Sirainen map->records = PTR_OFFSET(map->mmap_base, map->hdr->header_size);
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen map->records_count = map->hdr->messages_count;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen return 1;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen}
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainenstatic int mail_index_read_map(struct mail_index *index,
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct mail_index_map *map, int *retry_r)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen{
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen struct mail_index_header hdr, *hdrp;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen void *data = NULL;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen ssize_t ret;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen size_t pos, records_size;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen i_assert(map->mmap_base == NULL);
6df0ab0c1ab91f06b6418cb30eff44405a1b8f02Timo Sirainen
6df0ab0c1ab91f06b6418cb30eff44405a1b8f02Timo Sirainen *retry_r = FALSE;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen memset(&hdr, 0, sizeof(hdr));
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ret = 1;
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen for (pos = 0; ret > 0 && pos < sizeof(hdr); ) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen ret = pread(index->fd, PTR_OFFSET(&hdr, pos),
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen sizeof(hdr) - pos, pos);
e53ab6c7081246c865917f9aa0eff031a08ad1e7Timo Sirainen if (ret > 0)
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen pos += ret;
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen }
83bb013a99f0936995f9c7a1077822662d8fefdbTimo Sirainen
ceac44e7560fcbf6fc2f932c7b624a5055dc3bc9Timo Sirainen if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
ceac44e7560fcbf6fc2f932c7b624a5055dc3bc9Timo Sirainen (ret > 0 || pos >= hdr.base_header_size)) {
ceac44e7560fcbf6fc2f932c7b624a5055dc3bc9Timo Sirainen if (hdr.base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
ceac44e7560fcbf6fc2f932c7b624a5055dc3bc9Timo Sirainen hdr.header_size < hdr.base_header_size) {
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen mail_index_set_error(index, "Corrupted index file %s: "
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen "Corrupted header sizes (base %u, full %u)",
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen index->filepath, hdr.base_header_size,
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen hdr.header_size);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen return 0;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen }
f46885a5b78b15a8d2419f6e5d13b643bd85e41fTimo Sirainen
225e82df5dd1e765f4e52b80c954558f00e5a7dfTimo Sirainen buffer_reset(map->hdr_copy_buf);
73b50eecfc31750a312e2f940023f522eb07178cTimo Sirainen if (hdr.base_header_size < sizeof(hdr)) {
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen buffer_append_zero(map->hdr_copy_buf, sizeof(hdr) +
838e367716bbd5e44b4a1691db9cbf72af53e9f0Timo Sirainen hdr.header_size -
6564208826b0f46a00f010d1b5711d85944c3c88Timo Sirainen hdr.base_header_size);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen buffer_write(map->hdr_copy_buf, 0,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen &hdr, hdr.base_header_size);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* @UNSAFE */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen ret = pread_full(index->fd,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen PTR_OFFSET(map->hdr_copy_buf->data,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen sizeof(hdr)),
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen hdr.header_size - hdr.base_header_size,
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen hdr.base_header_size);
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen
13e130c3af3032982de6b1d13c6dcddda9164848Timo Sirainen hdrp = buffer_get_modifyable_data(map->hdr_copy_buf,
13e130c3af3032982de6b1d13c6dcddda9164848Timo Sirainen NULL);
13e130c3af3032982de6b1d13c6dcddda9164848Timo Sirainen hdrp->base_header_size = sizeof(hdr);
13e130c3af3032982de6b1d13c6dcddda9164848Timo Sirainen hdrp->header_size = map->hdr_copy_buf->used;
13e130c3af3032982de6b1d13c6dcddda9164848Timo Sirainen } else {
13e130c3af3032982de6b1d13c6dcddda9164848Timo Sirainen buffer_append(map->hdr_copy_buf, &hdr, pos);
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen buffer_append_zero(map->hdr_copy_buf,
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen hdr.header_size - pos);
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen /* @UNSAFE */
ecd69c4e8371853667e01b0c16d436ef7f7393e2Timo Sirainen ret = pread_full(index->fd,
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen PTR_OFFSET(map->hdr_copy_buf->data,
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen pos),
310767ca33e7636d40ec45dee68a2c319a5fa3c0Timo Sirainen hdr.header_size - pos, pos);
5f44975ec6c5755dd74bcd4c47a123a7242ecab3Timo Sirainen }
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen }
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen if (ret > 0) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen records_size = hdr.messages_count * hdr.record_size;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
e4c90f0b88e40a8f92b8f5e1f1a3ea701e5c965cTimo Sirainen if (map->buffer == NULL) {
defb12ecd360df672ffb2f4dbf4d1218a0a9549cTimo Sirainen map->buffer = buffer_create_dynamic(default_pool,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen records_size,
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen (size_t)-1);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen }
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* @UNSAFE */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen buffer_set_used_size(map->buffer, 0);
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen data = buffer_append_space_unsafe(map->buffer, records_size);
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen ret = pread_full(index->fd, data, records_size,
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen hdr.header_size);
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen }
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen if (ret < 0) {
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen if (errno == ESTALE) {
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen *retry_r = TRUE;
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen return 0;
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen }
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen mail_index_set_syscall_error(index, "pread_full()");
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen return -1;
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen }
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen if (ret == 0) {
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen mail_index_set_error(index,
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen "Corrupted index file %s: File too small",
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen index->filepath);
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
194603b35061fea1ee8d171a7104b6985c610966Timo Sirainen map->records = data;
194603b35061fea1ee8d171a7104b6985c610966Timo Sirainen map->records_count = hdr.messages_count;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen map->hdr = map->hdr_copy_buf->data;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen map->hdr_base = map->hdr_copy_buf->data;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen return 1;
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen}
50c617761ee9653bd44646a95178773a3686d62eTimo Sirainen
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainenstatic int mail_index_read_map_with_retry(struct mail_index *index,
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen struct mail_index_map *map)
cd83124e5d070a016c590bb0b1096d7828c7b6adTimo Sirainen{
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen int i, ret, retry;
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) {
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen ret = mail_index_read_map(index, map, &retry);
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen if (ret != 0 || !retry)
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen return ret;
c04f9a724a7b3cc649485a61b0a540868d25d71bTimo Sirainen
96308127e006bb3b1108093bcf4cc1fd9481cb7aTimo Sirainen /* ESTALE - reopen index file */
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen if (close(index->fd) < 0)
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen mail_index_set_syscall_error(index, "close()");
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen index->fd = -1;
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen ret = mail_index_try_open_only(index);
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen if (ret <= 0) {
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen if (ret == 0) {
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen /* the file was lost */
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen errno = ENOENT;
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen mail_index_set_syscall_error(index, "open()");
2d01cc1880cf2afd4fb1c8ad7fa6ce78e562e71eTimo Sirainen }
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen return -1;
1eaaa2c9003cf3fbf672d597473e3f84e70d2ee6Timo Sirainen }
900bb5e316d030cdebff7ee128ce65881dfb27f7Timo Sirainen }
900bb5e316d030cdebff7ee128ce65881dfb27f7Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen /* Too many ESTALE retries */
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen mail_index_set_syscall_error(index, "read_map()");
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return -1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainenstatic int mail_index_map_try_existing(struct mail_index_map *map)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
fdc557286bc9f92c5f3bb49096ff6e2bcec0ea79Timo Sirainen const struct mail_index_header *hdr;
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen size_t used_size;
c4b376dd6e0c423006d7ac83a39253bcaf8e7c47Timo Sirainen
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen return 0;
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen
d9076f5939edf5d20a261494b1a861dcbb0d32e2Timo Sirainen hdr = map->mmap_base;
e63bdfedcf61e1a9ee21990140cbd0d0638da7e1Timo Sirainen
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen /* always check corrupted-flag to avoid errors later */
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0)
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen return -1;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen used_size = hdr->header_size + hdr->messages_count * hdr->record_size;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen if (map->mmap_size >= used_size && map->hdr == hdr) {
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen map->records_count = hdr->messages_count;
3dd0679b6f24be0287cc42d7a60bbf59cdf8b637Timo Sirainen return 1;
a3ee5ce6ecc8e228ee69300fdd562d7ac8be89a7Timo Sirainen }
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen return 0;
f4616f1875297fb2f583d913c0f01b075bdecd5bTimo Sirainen}
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen
4321f6c969e7b8f6b243ff5bb6b8d297921676f6Timo Sirainenint mail_index_map(struct mail_index *index, int force)
4321f6c969e7b8f6b243ff5bb6b8d297921676f6Timo Sirainen{
d54ab8987e482a8df250615b44f41fa040c38741Timo Sirainen struct mail_index_map *map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen int ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen if (!force && index->map != NULL) {
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen ret = mail_index_map_try_existing(index->map);
e2d268e9531227ead6a98466ecf3c046c857ef70Timo Sirainen if (ret != 0)
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen return ret;
ad48319996942463675b53877092ab7e13a7a75aTimo Sirainen }
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen if (index->map != NULL && index->map->refcount > 1) {
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen /* this map is already used by some views and they may have
e3aeeb634245e80d4f643f8d2eea11d6b72336d8Timo Sirainen pointers into it. leave them and create a new mapping. */
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen index->map->refcount--;
0cce885512b836ce021260a58e7b4f099b36d0f1Timo Sirainen index->map = NULL;
9617ac7078a17bd346fed69526861c3e7fd9d809Timo Sirainen }
4d527c363482be2b65dd0573d878ecda86cbb0bbTimo Sirainen
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen map = index->map;
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen if (map == NULL) {
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen map = i_new(struct mail_index_map, 1);
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen map->refcount = 1;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen map->hdr_copy_buf =
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen buffer_create_dynamic(default_pool,
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen sizeof(*map->hdr), (size_t)-1);
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen if (map->write_to_disk) {
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen /* we have modified this mapping and it's waiting to
b9f564d00b7a115f465ffd6840341c7b8f9bfc8aTimo Sirainen be written to disk once we drop exclusive lock.
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen mapping couldn't have changed, so do nothing. */
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen return 1;
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen }
b62140c5849297a800fee942026d9c3cb8c60206Timo Sirainen /* FIXME: we need to re-read header */
b09eaeb9a81e5b58c6e605eb762573a2b4a69e0eTimo Sirainen } else if (map->mmap_base != NULL) {
b09eaeb9a81e5b58c6e605eb762573a2b4a69e0eTimo Sirainen i_assert(map->buffer == NULL);
b09eaeb9a81e5b58c6e605eb762573a2b4a69e0eTimo Sirainen if (munmap(map->mmap_base, map->mmap_size) < 0)
ba8ff75a149d6936f769a2d1dfceaab9da87863bTimo Sirainen mail_index_set_syscall_error(index, "munmap()");
ba8ff75a149d6936f769a2d1dfceaab9da87863bTimo Sirainen map->mmap_base = NULL;
abec3f4c5ec486c393e1513950f2d4819dcbc30fTimo Sirainen }
abec3f4c5ec486c393e1513950f2d4819dcbc30fTimo Sirainen
00aaafa8619b4e6aaf23f15d30c36f06ccb820bcTimo Sirainen index->hdr = NULL;
00aaafa8619b4e6aaf23f15d30c36f06ccb820bcTimo Sirainen index->map = NULL;
02f884382018f4435407374a059407caa44c9239Timo Sirainen
02f884382018f4435407374a059407caa44c9239Timo Sirainen if (!index->mmap_disable) {
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen if ((ret = mail_index_mmap(index, map)) <= 0) {
a423d985ba7261661475811c22b21b80ec765a71Timo Sirainen mail_index_unmap_forced(index, map);
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen return ret;
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen }
64b61cd24d630223478ccbe1934b9f60f0881f59Timo Sirainen } else {
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen if (mail_index_read_map_with_retry(index, map) < 0) {
d756ebcfa96bd7cff02097c8f26df9df368b81b1Timo Sirainen mail_index_unmap_forced(index, map);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen }
de58be41126e5d68008d2ea706d62ccdc1f29337Timo Sirainen
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen ret = mail_index_check_header(index, map);
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen if (ret < 0) {
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen mail_index_unmap_forced(index, map);
a4f09749814b93e8ad3ec8a0dc18885b874d6f8cTimo Sirainen return 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (ret == 0)
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen index->fsck = TRUE;
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen map->log_file_seq = map->hdr->log_file_seq;
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen map->log_file_offset = map->hdr->log_file_offset;
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainen map->base_header_size = map->hdr->base_header_size;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->hdr = map->hdr;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen index->map = map;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen return 1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen}
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
d6badc27cd6e8d3398877b6766cb0aaeef3a7800Timo Sirainenstruct mail_index_map *
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainenmail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size)
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen{
420040a5930a2b497e79ff0b5f59ba4b764a5b39Timo Sirainen struct mail_index_map *mem_map;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_header *hdr;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen struct mail_index_ext *extensions;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen void *src, *dest;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen size_t size, copy_size;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen unsigned int i, count;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen if (MAIL_INDEX_MAP_IS_IN_MEMORY(map) &&
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen map->hdr->record_size == new_record_size) {
1f1ee8db68d9ae1604350801cd8dc33ebe29fe8aTimo Sirainen map->refcount++;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen return map;
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen }
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen
268b72128fc3400912e9a6b83faf4950a367c2ffTimo Sirainen size = map->records_count * new_record_size;
268b72128fc3400912e9a6b83faf4950a367c2ffTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mem_map = i_new(struct mail_index_map, 1);
d9a129b491613014ce5f31fe1ab20903e2899ea4Timo Sirainen mem_map->refcount = 1;
d9a129b491613014ce5f31fe1ab20903e2899ea4Timo Sirainen mem_map->buffer = buffer_create_dynamic(default_pool, size, (size_t)-1);
d9a129b491613014ce5f31fe1ab20903e2899ea4Timo Sirainen if (map->hdr->record_size == new_record_size)
e15b305e90c9834734ccf35ed78f0ad29d570ee9Timo Sirainen buffer_append(mem_map->buffer, map->records, size);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen else {
910fa4e4204a73d3d24c03f3059dd24e727ca057Timo Sirainen copy_size = I_MIN(map->hdr->record_size, new_record_size);
7631f16156aca373004953fe6b01a7f343fb47e0Timo Sirainen src = map->records;
4bbd396aa6198c84f3f7763b6e8a63a26e97e141Timo Sirainen for (i = 0; i < map->records_count; i++) {
b83deefd2cf1e293373673eefb4d5cf60907978cTimo Sirainen dest = buffer_append_space_unsafe(mem_map->buffer,
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen new_record_size);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen memcpy(dest, src, copy_size);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen src = PTR_OFFSET(src, map->hdr->record_size);
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen }
546335814920fb6b5b44c68c7803e654eefeae9dTimo Sirainen }
546335814920fb6b5b44c68c7803e654eefeae9dTimo Sirainen
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen mem_map->records_count = map->records_count;
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen mem_map->hdr_copy_buf = buffer_create_dynamic(default_pool,
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen map->hdr->header_size,
ca98892a6b8a30ffc1fe26fcf02c7d59e3204e7eTimo Sirainen (size_t)-1);
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen buffer_append(mem_map->hdr_copy_buf, map->hdr, map->hdr->header_size);
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen
f46885a5b78b15a8d2419f6e5d13b643bd85e41fTimo Sirainen hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL);
f46885a5b78b15a8d2419f6e5d13b643bd85e41fTimo Sirainen hdr->record_size = new_record_size;
f46885a5b78b15a8d2419f6e5d13b643bd85e41fTimo Sirainen mem_map->hdr = hdr;
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen /* copy extensions */
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen if (map->ext_id_map != NULL) {
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen count = map->ext_id_map->used / sizeof(uint32_t);
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen mail_index_map_init_extbufs(mem_map, count);
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen buffer_append_buf(mem_map->extensions, map->extensions,
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen 0, (size_t)-1);
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen buffer_append_buf(mem_map->ext_id_map, map->ext_id_map,
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen 0, (size_t)-1);
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen /* fix the name pointers to use our own pool */
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen extensions = buffer_get_modifyable_data(mem_map->extensions,
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen NULL);
036626b19f14bef582f96e556913ae91b1d67881Timo Sirainen for (i = 0; i < count; i++) {
306b3f41b05da642d87e7ca7a1496efce9f5902fTimo Sirainen extensions[i].name = p_strdup(mem_map->extension_pool,
b3b4f3875850099c9292ad74d08bb385c3988f8fTimo Sirainen extensions[i].name);
7a2d48763b43c4fd8bc17c444271f0b802113a5fTimo Sirainen }
7a2d48763b43c4fd8bc17c444271f0b802113a5fTimo Sirainen }
7a2d48763b43c4fd8bc17c444271f0b802113a5fTimo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen return mem_map;
}
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r)
{
const uint32_t *id_map;
if (map->ext_id_map == NULL ||
map->ext_id_map->used / sizeof(*id_map) <= ext_id)
return 0;
id_map = map->ext_id_map->data;
*idx_r = id_map[ext_id];
return *idx_r != (uint32_t)-1;
}
static int mail_index_try_open_only(struct mail_index *index)
{
int i;
for (i = 0; i < 3; i++) {
index->fd = open(index->filepath, O_RDWR);
if (index->fd == -1 && errno == EACCES) {
index->fd = open(index->filepath, O_RDONLY);
index->readonly = TRUE;
}
if (index->fd != -1 || errno != ESTALE)
break;
/* May happen with some OSes with NFS. Try again, although
there's still a race condition with another computer
creating the index file again. However, we can't try forever
as ESTALE happens also if index directory has been deleted
from server.. */
}
if (index->fd == -1) {
if (errno != ENOENT)
return mail_index_set_syscall_error(index, "open()");
/* have to create it */
return 0;
}
return 1;
}
static int
mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
if (lock_id_r != NULL)
*lock_id_r = 0;
ret = mail_index_try_open_only(index);
if (ret <= 0)
return ret;
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) {
(void)close(index->fd);
index->fd = -1;
return -1;
}
ret = mail_index_map(index, FALSE);
if (ret == 0) {
/* it's corrupted - recreate it */
mail_index_unlock(index, lock_id);
if (lock_id_r != NULL)
*lock_id_r = 0;
(void)close(index->fd);
index->fd = -1;
} else {
if (lock_id_r != NULL)
*lock_id_r = lock_id;
else
mail_index_unlock(index, lock_id);
}
return ret;
}
int mail_index_write_base_header(struct mail_index *index,
const struct mail_index_header *hdr)
{
size_t hdr_size;
hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
memcpy(index->map->mmap_base, hdr, hdr_size);
if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
} else {
if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
mail_index_set_syscall_error(index, "pwrite_full()");
return -1;
}
buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size);
i_assert(index->hdr == index->map->hdr_copy_buf->data);
}
return 0;
}
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r)
{
const char *path;
int fd;
path = *path_r = t_strconcat(index->filepath, ".tmp", NULL);
fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode);
if (fd == -1)
return mail_index_file_set_syscall_error(index, path, "open()");
if (index->gid != (gid_t)-1 &&
fchown(index->fd, (uid_t)-1, index->gid) < 0) {
mail_index_file_set_syscall_error(index, path, "fchown()");
return -1;
}
return fd;
}
static int mail_index_create(struct mail_index *index,
struct mail_index_header *hdr)
{
const char *path;
uint32_t seq;
uoff_t offset;
int ret;
/* log file lock protects index creation */
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
ret = mail_index_try_open(index, NULL);
if (ret != 0) {
mail_transaction_log_sync_unlock(index->log);
return ret < 0 ? -1 : 0;
}
/* create it fully in index.tmp first */
index->fd = mail_index_create_tmp_file(index, &path);
if (index->fd == -1)
ret = -1;
else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
ret = -1;
} else {
ret = mail_index_map(index, FALSE);
}
if (ret == 0) {
/* it's corrupted even while we just created it,
should never happen unless someone pokes the file directly */
mail_index_set_error(index,
"Newly created index file is corrupted: %s", path);
ret = -1;
}
if (ret < 0) {
if (unlink(path) < 0 && errno != ENOENT) {
mail_index_file_set_syscall_error(index, path,
"unlink()");
}
} else {
/* make it visible to others */
if (rename(path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
path, index->filepath);
ret = -1;
}
}
mail_transaction_log_sync_unlock(index->log);
return ret;
}
static void mail_index_header_init(struct mail_index_header *hdr)
{
time_t now = time(NULL);
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
hdr->base_header_size = sizeof(*hdr);
hdr->header_size = sizeof(*hdr);
hdr->record_size = sizeof(struct mail_index_record);
hdr->keywords_mask_size = sizeof(keywords_mask_t);
#ifndef WORDS_BIGENDIAN
hdr->compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
#endif
hdr->compat_data[1] = sizeof(uoff_t);
hdr->compat_data[2] = sizeof(time_t);
hdr->indexid = now;
hdr->next_uid = 1;
}
/* returns -1 = error, 0 = won't create, 1 = ok */
static int mail_index_open_files(struct mail_index *index,
enum mail_index_open_flags flags)
{
struct mail_index_header hdr;
unsigned int lock_id = 0;
int ret;
ret = mail_index_try_open(index, &lock_id);
if (ret > 0)
hdr = *index->hdr;
else if (ret == 0) {
/* doesn't exist, or corrupted */
if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
return 0;
mail_index_header_init(&hdr);
index->hdr = &hdr;
} else if (ret < 0)
return -1;
index->indexid = hdr.indexid;
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL)
return -1;
if (index->fd == -1) {
if (lock_id != 0) {
mail_index_unlock(index, lock_id);
lock_id = 0;
}
if (mail_index_create(index, &hdr) < 0)
return -1;
}
if (lock_id == 0) {
if (mail_index_lock_shared(index, FALSE, &lock_id) < 0)
return -1;
}
index->cache = mail_cache_open_or_create(index);
if (index->cache == NULL)
return -1;
mail_index_unlock(index, lock_id);
return 1;
}
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags)
{
int i = 0, ret;
if (index->opened)
return 0;
index->filepath = i_strconcat(index->dir, "/", index->prefix, NULL);
do {
index->shared_lock_count = 0;
index->excl_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id = 2;
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
index->log_locked = FALSE;
index->mmap_disable =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0;
index->mmap_no_write =
(flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0;
index->fcntl_locks_disable =
(flags & MAIL_INDEX_OPEN_FLAG_FCNTL_LOCKS_DISABLE) != 0;
index->readonly = FALSE;
ret = mail_index_open_files(index, flags);
if (ret <= 0)
break;
index->opened = TRUE;
if (index->fsck) {
index->fsck = FALSE;
ret = mail_index_fsck(index);
if (ret == 0) {
/* completely broken, reopen */
if (i++ < 3)
continue;
/* too many tries */
ret = -1;
}
}
break;
} while (1);
if (ret <= 0)
mail_index_close(index);
return ret;
}
void mail_index_close(struct mail_index *index)
{
if (index->log != NULL) {
mail_transaction_log_close(index->log);
index->log = NULL;
}
if (index->map != NULL) {
mail_index_unmap(index, index->map);
index->map = NULL;
}
if (index->cache != NULL) {
mail_cache_free(index->cache);
index->cache = NULL;
}
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = -1;
}
i_free(index->copy_lock_path);
index->copy_lock_path = NULL;
i_free(index->filepath);
index->filepath = NULL;
index->indexid = 0;
index->opened = FALSE;
}
int mail_index_reopen(struct mail_index *index, int fd)
{
struct mail_index_map *old_map;
unsigned int old_shared_locks, old_lock_id, lock_id = 0;
int ret, old_fd, old_lock_type;
old_map = index->map;
old_fd = index->fd;
index->map = NULL;
index->hdr = NULL;
/* new file, new locks. the old fd can keep it's locks, they don't
matter anymore as no-one's going to modify the file. */
old_lock_type = index->lock_type;
old_lock_id = index->lock_id;
old_shared_locks = index->shared_lock_count;
if (index->lock_type == F_RDLCK)
index->lock_type = F_UNLCK;
index->lock_id += 2;
index->shared_lock_count = 0;
if (fd != -1) {
index->fd = fd;
ret = 0;
} else {
i_assert(index->excl_lock_count == 0);
ret = mail_index_try_open_only(index);
if (ret > 0)
ret = mail_index_lock_shared(index, FALSE, &lock_id);
else if (ret == 0) {
/* index file is lost */
ret = -1;
}
}
if (ret == 0) {
if (mail_index_map(index, FALSE) <= 0)
ret = -1;
}
if (lock_id != 0)
mail_index_unlock(index, lock_id);
if (ret == 0) {
mail_index_unmap(index, old_map);
if (close(old_fd) < 0)
mail_index_set_syscall_error(index, "close()");
} else {
if (index->map != NULL)
mail_index_unmap(index, index->map);
if (index->fd != -1) {
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
}
index->map = old_map;
index->hdr = index->map->hdr;
index->fd = old_fd;
index->lock_type = old_lock_type;
index->lock_id = old_lock_id;
index->shared_lock_count = old_shared_locks;
}
return ret;
}
int mail_index_refresh(struct mail_index *index)
{
struct stat st1, st2;
if (fstat(index->fd, &st1) < 0)
return mail_index_set_syscall_error(index, "fstat()");
if (stat(index->filepath, &st2) < 0) {
mail_index_set_syscall_error(index, "stat()");
if (errno != ENOENT)
return -1;
/* lost it? recreate */
(void)mail_index_mark_corrupted(index);
return -1;
}
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
struct mail_cache *mail_index_get_cache(struct mail_index *index)
{
return index->cache;
}
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
{
va_list va;
i_free(index->error);
if (fmt == NULL)
index->error = NULL;
else {
va_start(va, fmt);
index->error = i_strdup_vprintf(fmt, va);
va_end(va);
i_error("%s", index->error);
}
return -1;
}
void mail_index_set_inconsistent(struct mail_index *index)
{
index->indexid = 0;
}
void mail_index_mark_corrupted(struct mail_index *index)
{
struct mail_index_header hdr;
mail_index_set_inconsistent(index);
if (index->readonly)
return;
/* make sure we can write the header */
if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
if (mprotect(index->map->mmap_base, sizeof(hdr),
PROT_READ | PROT_WRITE) < 0) {
mail_index_set_syscall_error(index, "mprotect()");
return;
}
}
hdr = *index->hdr;
hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
if (mail_index_write_base_header(index, &hdr) == 0) {
if (fsync(index->fd) < 0)
mail_index_set_syscall_error(index, "fsync()");
}
}
int mail_index_set_syscall_error(struct mail_index *index,
const char *function)
{
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with index file %s: %m",
function, index->filepath);
}
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function)
{
i_assert(filepath != NULL);
i_assert(function != NULL);
if (ENOSPACE(errno)) {
index->nodiskspace = TRUE;
return -1;
}
return mail_index_set_error(index, "%s failed with file %s: %m",
function, filepath);
}
enum mail_index_error mail_index_get_last_error(struct mail_index *index)
{
if (index->nodiskspace)
return MAIL_INDEX_ERROR_DISKSPACE;
if (index->error != NULL)
return MAIL_INDEX_ERROR_INTERNAL;
return MAIL_INDEX_ERROR_NONE;
}
const char *mail_index_get_error_message(struct mail_index *index)
{
return index->error;
}
void mail_index_reset_error(struct mail_index *index)
{
if (index->error != NULL) {
i_free(index->error);
index->error = NULL;
}
index->nodiskspace = FALSE;
index->index_lock_timeout = FALSE;
}
uint32_t mail_index_uint32_to_offset(uint32_t offset)
{
unsigned char buf[4];
i_assert(offset < 0x40000000);
i_assert((offset & 3) == 0);
offset >>= 2;
buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
buf[3] = 0x80 | (offset & 0x0000007f);
return *((uint32_t *) buf);
}
uint32_t mail_index_offset_to_uint32(uint32_t offset)
{
const unsigned char *buf = (const unsigned char *) &offset;
if ((offset & 0x80808080) != 0x80808080)
return 0;
return (((uint32_t)buf[3] & 0x7f) << 2) |
(((uint32_t)buf[2] & 0x7f) << 9) |
(((uint32_t)buf[1] & 0x7f) << 16) |
(((uint32_t)buf[0] & 0x7f) << 23);
}