mail-cache-fields.c revision 9c4c535b86e9473ad97c6e9242ed84f3d9d69d0d
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2004-2007 Dovecot authors, see the included COPYING file */
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "lib.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "buffer.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "hash.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "file-cache.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "read-full.h"
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen#include "write-full.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "mmap-util.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include "mail-cache-private.h"
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen#include <stddef.h>
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen#define CACHE_FIELD_IS_NEWLY_WANTED(cache, field_idx) \
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen ((cache)->field_file_map[field_idx] == (uint32_t)-1 && \
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen (cache)->fields[field_idx].used)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainenstatic bool field_has_fixed_size(enum mail_cache_field_type type)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen{
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen switch (type) {
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen case MAIL_CACHE_FIELD_FIXED_SIZE:
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen case MAIL_CACHE_FIELD_BITMASK:
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return TRUE;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen case MAIL_CACHE_FIELD_VARIABLE_SIZE:
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen case MAIL_CACHE_FIELD_STRING:
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen case MAIL_CACHE_FIELD_HEADER:
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return FALSE;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen case MAIL_CACHE_FIELD_COUNT:
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen break;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen i_unreached();
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen return FALSE;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen}
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainenstatic bool field_decision_is_valid(enum mail_cache_decision_type type)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen{
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen switch (type & ~MAIL_CACHE_DECISION_FORCED) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen case MAIL_CACHE_DECISION_NO:
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen case MAIL_CACHE_DECISION_TEMP:
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen case MAIL_CACHE_DECISION_YES:
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return TRUE;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen default:
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return FALSE;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen}
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainenstatic int field_type_verify(struct mail_cache *cache, unsigned int idx,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen enum mail_cache_field_type type, unsigned int size)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen{
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen const struct mail_cache_field *field = &cache->fields[idx].field;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (field->type != type) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen mail_cache_set_corrupted(cache,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen "registered field %s type changed", field->name);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return -1;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (field->field_size != size && field_has_fixed_size(type)) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen mail_cache_set_corrupted(cache,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen "registered field %s size changed", field->name);
46b823ac3bce2c0f9f0fc73911e48d3a77b04fbeTimo Sirainen return -1;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return 0;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen}
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainenvoid mail_cache_register_fields(struct mail_cache *cache,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen struct mail_cache_field *fields,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen unsigned int fields_count)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen{
e245fb1302121d2bc2580f61e040c2c8a558ee9eTimo Sirainen void *orig_key, *orig_value;
e245fb1302121d2bc2580f61e040c2c8a558ee9eTimo Sirainen char *name;
e245fb1302121d2bc2580f61e040c2c8a558ee9eTimo Sirainen unsigned int new_idx;
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen unsigned int i, j;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen new_idx = cache->fields_count;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen for (i = 0; i < fields_count; i++) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (hash_lookup_full(cache->field_name_hash, fields[i].name,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen &orig_key, &orig_value)) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen i_assert(fields[i].type < MAIL_CACHE_FIELD_COUNT);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen fields[i].idx =
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen POINTER_CAST_TO(orig_value, unsigned int);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen (void)field_type_verify(cache, fields[i].idx,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen fields[i].type,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen fields[i].field_size);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen continue;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen /* check if the same header is being registered in the
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen same field array */
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen for (j = 0; j < i; j++) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (strcasecmp(fields[i].name, fields[j].name) == 0) {
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen fields[i].idx = fields[j].idx;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen break;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (j == i)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen fields[i].idx = new_idx++;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (new_idx == cache->fields_count)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen /* @UNSAFE */
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen cache->fields = i_realloc(cache->fields,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen cache->fields_count * sizeof(*cache->fields),
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen new_idx * sizeof(*cache->fields));
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen cache->field_file_map =
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen i_realloc(cache->field_file_map,
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen cache->fields_count * sizeof(*cache->field_file_map),
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen new_idx * sizeof(*cache->field_file_map));
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen for (i = 0; i < fields_count; i++) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen unsigned int idx = fields[i].idx;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (idx < cache->fields_count)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen continue;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen /* new index - save it */
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen name = p_strdup(cache->field_pool, fields[i].name);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen cache->fields[idx].field = fields[i];
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen cache->fields[idx].field.name = name;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen cache->field_file_map[idx] = (uint32_t)-1;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen if (!field_has_fixed_size(cache->fields[idx].field.type))
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen cache->fields[idx].field.field_size = (unsigned int)-1;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen hash_insert(cache->field_name_hash, name, POINTER_CAST(idx));
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen }
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen cache->fields_count = new_idx;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen}
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainenunsigned int
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainenmail_cache_register_lookup(struct mail_cache *cache, const char *name)
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen{
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen void *orig_key, *orig_value;
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen if (hash_lookup_full(cache->field_name_hash, name,
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen &orig_key, &orig_value))
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return POINTER_CAST_TO(orig_value, unsigned int);
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen else
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return (unsigned int)-1;
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen}
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainenconst struct mail_cache_field *
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainenmail_cache_register_get_list(struct mail_cache *cache, pool_t pool,
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen unsigned int *count_r)
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen{
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen struct mail_cache_field *list;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen unsigned int i;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen if (!cache->opened)
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen (void)mail_cache_open_and_verify(cache);
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen list = p_new(pool, struct mail_cache_field, cache->fields_count);
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen for (i = 0; i < cache->fields_count; i++) {
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen list[i] = cache->fields[i].field;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen list[i].name = p_strdup(pool, list[i].name);
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen }
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen *count_r = cache->fields_count;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return list;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen}
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainenstatic int mail_cache_header_fields_get_offset(struct mail_cache *cache,
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen uint32_t *offset_r, bool map)
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen{
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen const struct mail_cache_header_fields *field_hdr;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen struct mail_cache_header_fields tmp_field_hdr;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen uint32_t offset = 0, next_offset;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen unsigned int next_count = 0;
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen int ret;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen if (MAIL_CACHE_IS_UNUSABLE(cache)) {
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen *offset_r = 0;
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen return 0;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen }
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen /* find the latest header */
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen offset = 0;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen next_offset = cache->last_field_header_offset != 0 ?
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen cache->last_field_header_offset :
6adf683655750bcb809275cd65dc75fd12214198Timo Sirainen mail_index_offset_to_uint32(cache->hdr->field_header_offset);
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen while (next_offset != 0) {
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen if (next_offset == offset) {
46631c1d903c409444b1b1c4a1d41a033c09ee37Timo Sirainen mail_cache_set_corrupted(cache,
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen "next_offset in field header loops");
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return -1;
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen }
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen offset = next_offset;
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen if (cache->mmap_base != NULL) {
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen if (mail_cache_map(cache, offset,
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen sizeof(*field_hdr)) < 0)
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen return -1;
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen field_hdr = CONST_PTR_OFFSET(cache->data, offset);
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen } else {
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen /* if we need to follow multiple offsets to get to
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen the last one, it's faster to just pread() the file
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen instead of going through cache */
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen ret = pread_full(cache->fd, &tmp_field_hdr,
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen sizeof(tmp_field_hdr), offset);
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen if (ret < 0) {
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen mail_cache_set_syscall_error(cache, "pread()");
d5e839aea288aceaddae28a1578cebda3c9e3b58Timo Sirainen return -1;
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen }
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen if (ret == 0) {
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen mail_cache_set_corrupted(cache,
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen "next_offset points outside file");
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen return -1;
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen }
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen field_hdr = &tmp_field_hdr;
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen }
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen next_offset =
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen mail_index_offset_to_uint32(field_hdr->next_offset);
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen next_count++;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen }
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen if (offset == 0) {
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen mail_cache_set_corrupted(cache, "missing header fields");
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen return -1;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen }
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen cache->last_field_header_offset = offset;
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT)
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen cache->need_compress_file_seq = cache->hdr->file_seq;
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen if (map) {
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen if (cache->file_cache != NULL) {
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen /* we can't trust that the cached data is valid */
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen file_cache_invalidate(cache->file_cache, offset,
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen field_hdr->size);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (mail_cache_map(cache, offset, field_hdr->size) < 0)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return -1;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen *offset_r = offset;
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return 0;
fb176cdc122707cda985ab3c09c02ccf3cec0af1Timo Sirainen}
153ed0fbca1f5f944b70937dfd71911db172ca97Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainenint mail_cache_header_fields_read(struct mail_cache *cache)
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen{
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen const struct mail_cache_header_fields *field_hdr = NULL;
8a8a3b43987b5ade914f22765e51c9e3de8179d3Timo Sirainen struct mail_cache_field field;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen const uint32_t *last_used, *sizes;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen const uint8_t *types, *decisions;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen const char *p, *names, *end;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen void *orig_key, *orig_value;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen unsigned int new_fields_count;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen time_t max_drop_time;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen uint32_t offset, i;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen if (mail_cache_header_fields_get_offset(cache, &offset, TRUE) < 0)
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen return -1;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen if (offset == 0) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen /* no fields - the file is empty */
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return 0;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen field_hdr = CONST_PTR_OFFSET(cache->data, offset);
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (offset + field_hdr->size > cache->mmap_length) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen mail_cache_set_corrupted(cache,
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen "field header points outside file");
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return -1;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen /* check the fixed size of the header. name[] has to be checked
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen separately */
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen if (field_hdr->size < sizeof(*field_hdr) +
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen field_hdr->fields_count * (sizeof(uint32_t)*2 + 1 + 2)) {
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen mail_cache_set_corrupted(cache, "invalid field header size");
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen return -1;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen field_hdr = CONST_PTR_OFFSET(cache->data, offset);
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen new_fields_count = field_hdr->fields_count;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen if (new_fields_count != 0) {
e44028b5df7045dd9e7f324175e73e3ff490cb5dTimo Sirainen cache->file_field_map =
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen i_realloc(cache->file_field_map,
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen cache->file_fields_count *
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen sizeof(unsigned int),
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen new_fields_count * sizeof(unsigned int));
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen } else {
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen i_free_and_null(cache->file_field_map);
b3f4c31f1533e25380f49f77d5bb1251bf43db2aTimo Sirainen }
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen cache->file_fields_count = new_fields_count;
bace943c67e6cd14ce6c994f533d82a3caad5bf1Timo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED());
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen sizes = CONST_PTR_OFFSET(field_hdr,
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count));
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen types = CONST_PTR_OFFSET(field_hdr,
c096257fbdaf4b9fcf8eb97aae94afdbb4e71ed4Timo Sirainen MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count));
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen decisions = CONST_PTR_OFFSET(field_hdr,
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count));
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen names = CONST_PTR_OFFSET(field_hdr,
e44028b5df7045dd9e7f324175e73e3ff490cb5dTimo Sirainen MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count));
e44028b5df7045dd9e7f324175e73e3ff490cb5dTimo Sirainen end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);
e44028b5df7045dd9e7f324175e73e3ff490cb5dTimo Sirainen
baf346e71ebd7b44fcba4b48f4d39845453b778bTimo Sirainen /* clear the old mapping */
bace943c67e6cd14ce6c994f533d82a3caad5bf1Timo Sirainen for (i = 0; i < cache->fields_count; i++)
bace943c67e6cd14ce6c994f533d82a3caad5bf1Timo Sirainen cache->field_file_map[i] = (uint32_t)-1;
c096257fbdaf4b9fcf8eb97aae94afdbb4e71ed4Timo Sirainen
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen max_drop_time = cache->index->map->hdr.day_stamp -
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen MAIL_CACHE_FIELD_DROP_SECS;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen memset(&field, 0, sizeof(field));
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen for (i = 0; i < field_hdr->fields_count; i++) {
c096257fbdaf4b9fcf8eb97aae94afdbb4e71ed4Timo Sirainen for (p = names; p != end && *p != '\0'; p++) ;
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen if (p == end || *names == '\0') {
2f4f603d4cebab2cc956c72164efb02da83515c5Timo Sirainen mail_cache_set_corrupted(cache,
1a0ece3e873e3864269ed7eaed957dc10c56d25fTimo Sirainen "field header names corrupted");
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return -1;
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen }
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen if (types[i] > MAIL_CACHE_FIELD_COUNT) {
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen mail_cache_set_corrupted(cache, "field type corrupted");
cf636afb3826f0d8e15c248aa1fc04ce72820e08Timo Sirainen return -1;
}
if (!field_decision_is_valid(decisions[i])) {
mail_cache_set_corrupted(cache,
"field decision type corrupted");
return -1;
}
if (hash_lookup_full(cache->field_name_hash, names,
&orig_key, &orig_value)) {
/* already exists, see if decision can be updated */
field.idx = POINTER_CAST_TO(orig_value, unsigned int);
if (!cache->fields[field.idx].decision_dirty) {
cache->fields[field.idx].field.decision =
decisions[i];
}
if (field_type_verify(cache, field.idx,
types[i], sizes[i]) < 0)
return -1;
} else {
field.name = names;
field.type = types[i];
field.field_size = sizes[i];
field.decision = decisions[i];
mail_cache_register_fields(cache, &field, 1);
}
if (cache->field_file_map[field.idx] != (uint32_t)-1) {
mail_cache_set_corrupted(cache,
"Duplicated field in header: %s", names);
return -1;
}
cache->fields[field.idx].used = TRUE;
cache->field_file_map[field.idx] = i;
cache->file_field_map[i] = field.idx;
/* update last_used if it's newer than ours */
if (last_used[i] > cache->fields[field.idx].last_used)
cache->fields[field.idx].last_used = last_used[i];
if (cache->fields[field.idx].last_used < max_drop_time &&
cache->fields[field.idx].last_used != 0) {
/* time to drop this field. don't bother dropping
fields that have never been used. */
cache->need_compress_file_seq = cache->hdr->file_seq;
}
names = p + 1;
}
return 0;
}
static void copy_to_buf(struct mail_cache *cache, buffer_t *dest, bool add_new,
size_t offset, size_t size)
{
const void *data;
unsigned int i, field;
/* copy the existing fields */
for (i = 0; i < cache->file_fields_count; i++) {
field = cache->file_field_map[i];
data = CONST_PTR_OFFSET(&cache->fields[field], offset);
buffer_append(dest, data, size);
}
if (!add_new)
return;
/* copy newly wanted fields */
for (i = 0; i < cache->fields_count; i++) {
if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
data = CONST_PTR_OFFSET(&cache->fields[i], offset);
buffer_append(dest, data, size);
}
}
}
static void copy_to_buf_byte(struct mail_cache *cache, buffer_t *dest,
bool add_new, size_t offset)
{
const int *data;
unsigned int i, field;
uint8_t byte;
/* copy the existing fields */
for (i = 0; i < cache->file_fields_count; i++) {
field = cache->file_field_map[i];
data = CONST_PTR_OFFSET(&cache->fields[field], offset);
byte = (uint8_t)*data;
buffer_append(dest, &byte, 1);
}
if (!add_new)
return;
/* copy newly wanted fields */
for (i = 0; i < cache->fields_count; i++) {
if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
data = CONST_PTR_OFFSET(&cache->fields[i], offset);
byte = (uint8_t)*data;
buffer_append(dest, &byte, 1);
}
}
}
static int mail_cache_header_fields_update_locked(struct mail_cache *cache)
{
buffer_t *buffer;
uint32_t i, offset, dec_offset;
int ret = 0;
if (mail_cache_header_fields_read(cache) < 0 ||
mail_cache_header_fields_get_offset(cache, &offset, FALSE) < 0)
return -1;
t_push();
buffer = buffer_create_dynamic(pool_datastack_create(), 256);
copy_to_buf(cache, buffer, FALSE,
offsetof(struct mail_cache_field_private, last_used),
sizeof(uint32_t));
ret = mail_cache_write(cache, buffer->data, buffer->used,
offset + MAIL_CACHE_FIELD_LAST_USED());
if (ret == 0) {
buffer_set_used_size(buffer, 0);
copy_to_buf_byte(cache, buffer, FALSE,
offsetof(struct mail_cache_field, decision));
dec_offset = offset +
MAIL_CACHE_FIELD_DECISION(cache->file_fields_count);
ret = mail_cache_write(cache, buffer->data, buffer->used,
dec_offset);
if (ret == 0) {
for (i = 0; i < cache->file_fields_count; i++)
cache->fields[i].decision_dirty = FALSE;
}
}
t_pop();
if (ret == 0)
cache->field_header_write_pending = FALSE;
return ret;
}
int mail_cache_header_fields_update(struct mail_cache *cache)
{
int ret;
if (cache->locked)
return mail_cache_header_fields_update_locked(cache);
if (mail_cache_lock(cache, NULL) <= 0)
return -1;
ret = mail_cache_header_fields_update_locked(cache);
if (mail_cache_unlock(cache) < 0)
ret = -1;
return ret;
}
void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest)
{
struct mail_cache_header_fields hdr;
unsigned int field;
const char *name;
uint32_t i;
memset(&hdr, 0, sizeof(hdr));
hdr.fields_count = cache->file_fields_count;
for (i = 0; i < cache->fields_count; i++) {
if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i))
hdr.fields_count++;
}
buffer_append(dest, &hdr, sizeof(hdr));
/* we have to keep the field order for the existing fields. */
copy_to_buf(cache, dest, TRUE,
offsetof(struct mail_cache_field_private, last_used),
sizeof(uint32_t));
copy_to_buf(cache, dest, TRUE,
offsetof(struct mail_cache_field, field_size),
sizeof(uint32_t));
copy_to_buf_byte(cache, dest, TRUE,
offsetof(struct mail_cache_field, type));
copy_to_buf_byte(cache, dest, TRUE,
offsetof(struct mail_cache_field, decision));
i_assert(dest->used == sizeof(hdr) +
(sizeof(uint32_t)*2 + 2) * hdr.fields_count);
/* add existing fields' names */
for (i = 0; i < cache->file_fields_count; i++) {
field = cache->file_field_map[i];
name = cache->fields[field].field.name;
buffer_append(dest, name, strlen(name)+1);
}
/* add newly wanted fields' names */
for (i = 0; i < cache->fields_count; i++) {
if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
name = cache->fields[i].field.name;
buffer_append(dest, name, strlen(name)+1);
}
}
hdr.size = dest->used;
buffer_write(dest, 0, &hdr, sizeof(hdr));
if ((hdr.size & 3) != 0)
buffer_append_zero(dest, 4 - (hdr.size & 3));
}
int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
uint32_t *offset_r)
{
if (mail_cache_header_fields_get_offset(cache, offset_r, FALSE) < 0)
return -1;
if (*offset_r == 0) {
*offset_r = offsetof(struct mail_cache_header,
field_header_offset);
} else {
*offset_r += offsetof(struct mail_cache_header_fields,
next_offset);
}
return 0;
}