mail-cache-fields.c revision 9dba6a3234ffd20d3c0260897b845b58399067f9
02c335c23bf5fa225a467c19f2c063fb0dc7b8c3Timo Sirainen/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "lib.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "ioloop.h"
0536ccb51d41e3078c3a9fa33e509fb4b2420f95Timo Sirainen#include "buffer.h"
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen#include "hash.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "file-cache.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "read-full.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "write-full.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mmap-util.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include "mail-cache-private.h"
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen#include <stddef.h>
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen#define CACHE_FIELD_IS_NEWLY_WANTED(cache, field_idx) \
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen ((cache)->field_file_map[field_idx] == (uint32_t)-1 && \
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen (cache)->fields[field_idx].used)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenstatic bool field_has_fixed_size(enum mail_cache_field_type type)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen switch (type) {
8cb72c59d5ea4e9e5f638d7ec840bb853f5a188eTimo Sirainen case MAIL_CACHE_FIELD_FIXED_SIZE:
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen case MAIL_CACHE_FIELD_BITMASK:
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen return TRUE;
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen case MAIL_CACHE_FIELD_VARIABLE_SIZE:
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen case MAIL_CACHE_FIELD_STRING:
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen case MAIL_CACHE_FIELD_HEADER:
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen return FALSE;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen case MAIL_CACHE_FIELD_COUNT:
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen break;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen }
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen i_unreached();
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen return FALSE;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen}
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainenstatic bool field_decision_is_valid(enum mail_cache_decision_type type)
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen{
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen switch (type & ~MAIL_CACHE_DECISION_FORCED) {
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen case MAIL_CACHE_DECISION_NO:
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen case MAIL_CACHE_DECISION_TEMP:
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen case MAIL_CACHE_DECISION_YES:
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen return TRUE;
862ec874f9373e3e499e237d3b9f71fdf1413feeTimo Sirainen default:
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen return FALSE;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen }
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen}
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainenstatic int field_type_verify(struct mail_cache *cache, unsigned int idx,
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen enum mail_cache_field_type type, unsigned int size)
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen{
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen const struct mail_cache_field *field = &cache->fields[idx].field;
59e26ff34b05cd971a111f8a42fc60c13d9f688bTimo Sirainen
9a48c2243fe98ca8393be7908f84d20c634bcdf9Timo Sirainen if (field->type != type) {
2d2ebe91d56e9a158de000c9d0026f65600fbcfaTimo Sirainen mail_cache_set_corrupted(cache,
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen "registered field %s type changed", field->name);
f988b93c2ef773987bcdcbfb4cca39b955e3a392Timo Sirainen return -1;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen }
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen if (field->field_size != size && field_has_fixed_size(type)) {
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen mail_cache_set_corrupted(cache,
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen "registered field %s size changed", field->name);
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen return -1;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen }
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen return 0;
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen}
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainenstatic void
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainenmail_cache_field_update(struct mail_cache *cache,
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen const struct mail_cache_field *newfield)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen struct mail_cache_field_private *orig;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen bool initial_registering;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen i_assert(newfield->type < MAIL_CACHE_FIELD_COUNT);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
571fd6ff94570ee11a72a20b649acfdac2495919Timo Sirainen /* are we still doing the initial cache field registering for
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen internal fields and for mail_*cache_fields settings? */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen initial_registering = cache->file_fields_count == 0;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen orig = &cache->fields[newfield->idx];
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if ((newfield->decision & MAIL_CACHE_DECISION_FORCED) != 0 ||
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen ((orig->field.decision & MAIL_CACHE_DECISION_FORCED) == 0 &&
5a1b498b646b5c5dbd1b3f3861df766f560578c5Timo Sirainen newfield->decision > orig->field.decision)) {
5a1b498b646b5c5dbd1b3f3861df766f560578c5Timo Sirainen orig->field.decision = newfield->decision;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (!initial_registering)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen orig->decision_dirty = TRUE;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen }
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen if (orig->field.last_used < newfield->last_used) {
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen orig->field.last_used = newfield->last_used;
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen if (!initial_registering)
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen orig->decision_dirty = TRUE;
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen }
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen if (orig->decision_dirty)
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen cache->field_header_write_pending = TRUE;
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen (void)field_type_verify(cache, newfield->idx,
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen newfield->type, newfield->field_size);
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen}
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainenvoid mail_cache_register_fields(struct mail_cache *cache,
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen struct mail_cache_field *fields,
eb98a038ca8b0ef33d1d11794803ce09547496faTimo Sirainen unsigned int fields_count)
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen{
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen char *name;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen void *value;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen unsigned int new_idx;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen unsigned int i, j, registered_count;
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen new_idx = cache->fields_count;
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen for (i = 0; i < fields_count; i++) {
211ed7806d8715ec2280ffbf5d10f0d6e4f1beb2Timo Sirainen if (hash_table_lookup_full(cache->field_name_hash,
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen fields[i].name, &name, &value)) {
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen fields[i].idx = POINTER_CAST_TO(value, unsigned int);
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen mail_cache_field_update(cache, &fields[i]);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen continue;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen }
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen /* check if the same header is being registered in the
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen same field array */
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen for (j = 0; j < i; j++) {
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen if (strcasecmp(fields[i].name, fields[j].name) == 0) {
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen fields[i].idx = fields[j].idx;
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen break;
59151b71059df1190acd75d8717ed04a7920c862Timo Sirainen }
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (j == i)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen fields[i].idx = new_idx++;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen }
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen if (new_idx == cache->fields_count)
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen return;
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen
556f95092c3bc850517d5ab2bb502024a55645f1Timo Sirainen /* @UNSAFE */
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen cache->fields = i_realloc_type(cache->fields,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen struct mail_cache_field_private,
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen cache->fields_count, new_idx);
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen cache->field_file_map =
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen i_realloc_type(cache->field_file_map, uint32_t,
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen cache->fields_count, new_idx);
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen registered_count = cache->fields_count;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen for (i = 0; i < fields_count; i++) {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen unsigned int idx = fields[i].idx;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen if (idx < registered_count)
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen continue;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen /* new index - save it */
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen name = p_strdup(cache->field_pool, fields[i].name);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen cache->fields[idx].field = fields[i];
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->fields[idx].field.name = name;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen cache->fields[idx].field.last_used = fields[i].last_used;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen cache->field_file_map[idx] = (uint32_t)-1;
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen if (!field_has_fixed_size(cache->fields[idx].field.type))
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen cache->fields[idx].field.field_size = UINT_MAX;
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen hash_table_insert(cache->field_name_hash, name,
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen POINTER_CAST(idx));
10ff47d5d6146995e16da00d36eca7d162064a7bTimo Sirainen registered_count++;
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen }
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen i_assert(registered_count == new_idx);
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen cache->fields_count = new_idx;
8f32e59200da904613506f5649ffa4d9f5989cebTimo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainenunsigned int
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainenmail_cache_register_lookup(struct mail_cache *cache, const char *name)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen char *key;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen void *value;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (hash_table_lookup_full(cache->field_name_hash, name, &key, &value))
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen return POINTER_CAST_TO(value, unsigned int);
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen else
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen return UINT_MAX;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenconst struct mail_cache_field *
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainenmail_cache_register_get_field(struct mail_cache *cache, unsigned int field_idx)
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen{
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainen i_assert(field_idx < cache->fields_count);
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen return &cache->fields[field_idx].field;
c9bf63e9094761767a63ac6b189bcf60bcffdc44Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainenstruct mail_cache_field *
5238111c460098d9cc8cc22527026138a278b9a4Timo Sirainenmail_cache_register_get_list(struct mail_cache *cache, pool_t pool,
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen unsigned int *count_r)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen struct mail_cache_field *list;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen unsigned int i;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen if (!cache->opened)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen (void)mail_cache_open_and_verify(cache);
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen list = cache->fields_count == 0 ? NULL :
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen p_new(pool, struct mail_cache_field, cache->fields_count);
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen for (i = 0; i < cache->fields_count; i++) {
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen list[i] = cache->fields[i].field;
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen list[i].name = p_strdup(pool, list[i].name);
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen }
de954ff15b495be13007a8aca2c09fd1d356a283Timo Sirainen
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen *count_r = cache->fields_count;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen return list;
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen}
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainenstatic int
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainenmail_cache_header_fields_get_offset(struct mail_cache *cache,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen uint32_t *offset_r,
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen const struct mail_cache_header_fields **field_hdr_r)
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen const struct mail_cache_header_fields *field_hdr;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen struct mail_cache_header_fields tmp_field_hdr;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen const void *data;
8f32e59200da904613506f5649ffa4d9f5989cebTimo Sirainen uint32_t offset = 0, next_offset, field_hdr_size;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen unsigned int next_count = 0;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen int ret;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen if (MAIL_CACHE_IS_UNUSABLE(cache)) {
b92e979748a22925b0770d3004eaab043ed69574Timo Sirainen *offset_r = 0;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen if (field_hdr_r != NULL)
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen *field_hdr_r = NULL;
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen return 0;
35283613d4c04ce18836e9fc431582c87b3710a0Timo Sirainen }
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen /* find the latest header */
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen offset = 0;
35283613d4c04ce18836e9fc431582c87b3710a0Timo Sirainen next_offset = cache->last_field_header_offset != 0 ?
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen cache->last_field_header_offset :
2767104d81e97a109f0aa9758792bfa1da325a97Timo Sirainen mail_index_offset_to_uint32(cache->hdr->field_header_offset);
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen while (next_offset != 0) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen if (next_offset == offset) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen mail_cache_set_corrupted(cache,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen "next_offset in field header loops");
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen return -1;
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen }
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen /* In Dovecot v2.2+ we don't try to use any holes,
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen so next_offset must always be larger than current offset.
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen also makes it easier to guarantee there aren't any loops
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen (which we don't bother doing for old files) */
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen if (next_offset < offset && cache->hdr->minor_version != 0) {
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen mail_cache_set_corrupted(cache,
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen "next_offset in field header decreases");
0dbcf4026ff8471b4d6d2e14f7e52321396bf087Timo Sirainen return -1;
0dbcf4026ff8471b4d6d2e14f7e52321396bf087Timo Sirainen }
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen offset = next_offset;
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen if (cache->mmap_base != NULL || cache->map_with_read) {
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen ret = mail_cache_map(cache, offset, sizeof(*field_hdr),
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen &data);
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen if (ret <= 0) {
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen if (ret < 0)
8b2cf1c1bd8ddcea0525b62fd35ba76e136828a1Timo Sirainen return -1;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen mail_cache_set_corrupted(cache,
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen "header field next_offset points outside file");
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen return -1;
6cc0546c058f3e6253c6f99727b28dd602712974Timo Sirainen }
0ce5f96804e81cb0f857e7df32c0272f1eed9377Timo Sirainen field_hdr = data;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen } else {
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen /* if we need to follow multiple offsets to get to
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen the last one, it's faster to just pread() the file
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen instead of going through cache */
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen ret = pread_full(cache->fd, &tmp_field_hdr,
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen sizeof(tmp_field_hdr), offset);
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen if (ret < 0) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen mail_cache_set_syscall_error(cache, "pread()");
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen return -1;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen }
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen if (ret == 0) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen mail_cache_set_corrupted(cache,
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen "header field next_offset points outside file");
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen return -1;
b92e979748a22925b0770d3004eaab043ed69574Timo Sirainen }
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen field_hdr = &tmp_field_hdr;
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen }
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen
555ebb032f9b8f0cdb66f27ce7374734833e7cacTimo Sirainen next_offset =
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_index_offset_to_uint32(field_hdr->next_offset);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen next_count++;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (offset == 0) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_cache_set_corrupted(cache, "missing header fields");
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return -1;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
4e8d6d03c2ff85448df79b181a2ea850fb5d4199Timo Sirainen cache->last_field_header_offset = offset;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (next_count > cache->index->optimization_set.cache.compress_header_continue_count)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen cache->need_compress_file_seq = cache->hdr->file_seq;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (field_hdr_r != NULL) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen /* detect corrupted size later */
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen field_hdr_size = I_MAX(field_hdr->size, sizeof(*field_hdr));
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen if (cache->file_cache != NULL) {
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen /* invalidate the cache fields area to make sure we
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen get the latest cache decisions/last_used fields */
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen file_cache_invalidate(cache->file_cache, offset,
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen field_hdr_size);
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (cache->read_buf != NULL)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen buffer_set_used_size(cache->read_buf, 0);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen ret = mail_cache_map(cache, offset, field_hdr_size, &data);
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (ret < 0)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return -1;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (ret == 0) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen mail_cache_set_corrupted(cache,
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen "header field size outside file");
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return -1;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen *field_hdr_r = data;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen *offset_r = offset;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return 0;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen}
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainenint mail_cache_header_fields_read(struct mail_cache *cache)
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen{
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen const struct mail_cache_header_fields *field_hdr;
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen struct mail_cache_field field;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen const uint32_t *last_used, *sizes;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen const uint8_t *types, *decisions;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen const char *p, *names, *end;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen char *orig_key;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen void *orig_value;
9de176ef7f3d28ff486c2a8805110b84389e4f19Timo Sirainen unsigned int fidx, new_fields_count;
f988b93c2ef773987bcdcbfb4cca39b955e3a392Timo Sirainen enum mail_cache_decision_type dec;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen time_t max_drop_time;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen uint32_t offset, i;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (mail_cache_header_fields_get_offset(cache, &offset, &field_hdr) < 0)
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen return -1;
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen if (offset == 0) {
98a73e104c7b9c3747053e63113451e24daf7f36Timo Sirainen /* no fields - the file is empty */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen return 0;
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen }
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen /* check the fixed size of the header. name[] has to be checked
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen separately */
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen if (field_hdr->fields_count > INT_MAX / MAIL_CACHE_FIELD_NAMES(1) ||
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen field_hdr->size < MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count)) {
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen mail_cache_set_corrupted(cache, "invalid field header size");
c0435c854a0e7246373b9752d163095cc4fbe985Timo Sirainen return -1;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen }
252db51b6c0a605163326b3ea5d09e9936ca3b29Timo Sirainen
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen new_fields_count = field_hdr->fields_count;
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen if (new_fields_count != 0) {
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen cache->file_field_map =
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen i_realloc_type(cache->file_field_map, unsigned int,
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen cache->file_fields_count, new_fields_count);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen } else {
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen i_free_and_null(cache->file_field_map);
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen }
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen cache->file_fields_count = new_fields_count;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
63866b7a81355543832d3fe01cd744ddd4ea197bTimo Sirainen last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED());
63866b7a81355543832d3fe01cd744ddd4ea197bTimo Sirainen sizes = CONST_PTR_OFFSET(field_hdr,
63866b7a81355543832d3fe01cd744ddd4ea197bTimo Sirainen MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count));
63866b7a81355543832d3fe01cd744ddd4ea197bTimo Sirainen types = CONST_PTR_OFFSET(field_hdr,
63866b7a81355543832d3fe01cd744ddd4ea197bTimo Sirainen MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count));
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen decisions = CONST_PTR_OFFSET(field_hdr,
989cafb9d84d8c98d6441fc1ab45b4c37762a98aTimo Sirainen MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count));
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen names = CONST_PTR_OFFSET(field_hdr,
ecc81625167ed96c04c02aa190a1ea5baa65b474Timo Sirainen MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count));
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen i_assert(names <= end);
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen /* clear the old mapping */
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen for (i = 0; i < cache->fields_count; i++)
3d77cc0d502dc69ffe2afe318605964dd40b7b20Timo Sirainen cache->field_file_map[i] = (uint32_t)-1;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen max_drop_time = cache->index->map->hdr.day_stamp == 0 ? 0 :
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen cache->index->map->hdr.day_stamp -
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen cache->index->optimization_set.cache.unaccessed_field_drop_secs;
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen i_zero(&field);
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen for (i = 0; i < field_hdr->fields_count; i++) {
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen for (p = names; p != end && *p != '\0'; p++) ;
d66ef20c30fee728899ee168c75fcc5ff8fbdac1Timo Sirainen if (p == end || *names == '\0') {
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen mail_cache_set_corrupted(cache,
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen "field header names corrupted");
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen return -1;
8cdb3234fe3c77e477c7a0e6934678f58fc54d4dTimo Sirainen }
46631c1d903c409444b1b1c4a1d41a033c09ee37Timo Sirainen
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen if (types[i] > MAIL_CACHE_FIELD_COUNT) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen mail_cache_set_corrupted(cache, "field type corrupted");
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen return -1;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen }
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen if (!field_decision_is_valid(decisions[i])) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen mail_cache_set_corrupted(cache,
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen "field decision type corrupted");
46631c1d903c409444b1b1c4a1d41a033c09ee37Timo Sirainen return -1;
3858a7a5da361c35f1e6e50c8e3214dc0cf379d6Phil Carmody }
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen if (hash_table_lookup_full(cache->field_name_hash, names,
db3ebf659c17fc5ca348a997e6311d20cfbb78e9Timo Sirainen &orig_key, &orig_value)) {
db3ebf659c17fc5ca348a997e6311d20cfbb78e9Timo Sirainen /* already exists, see if decision can be updated */
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen fidx = POINTER_CAST_TO(orig_value, unsigned int);
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen if (!cache->fields[fidx].decision_dirty &&
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen (cache->fields[fidx].field.decision &
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen MAIL_CACHE_DECISION_FORCED) == 0) {
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen cache->fields[fidx].field.decision =
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen decisions[i] & ~MAIL_CACHE_DECISION_FORCED;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen }
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen if (field_type_verify(cache, fidx,
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen types[i], sizes[i]) < 0)
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen return -1;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen } else {
46631c1d903c409444b1b1c4a1d41a033c09ee37Timo Sirainen field.name = names;
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen field.type = types[i];
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen field.field_size = sizes[i];
8ab3d88c56dff2f567193f80cc29821a64e576d1Timo Sirainen field.decision = decisions[i] & ~MAIL_CACHE_DECISION_FORCED;
71da447014454c84828d9dface77219875554d7dTimo Sirainen mail_cache_register_fields(cache, &field, 1);
71da447014454c84828d9dface77219875554d7dTimo Sirainen fidx = field.idx;
71da447014454c84828d9dface77219875554d7dTimo Sirainen }
71da447014454c84828d9dface77219875554d7dTimo Sirainen if (cache->field_file_map[fidx] != (uint32_t)-1) {
71da447014454c84828d9dface77219875554d7dTimo Sirainen mail_cache_set_corrupted(cache,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen "Duplicated field in header: %s", names);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return -1;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen cache->fields[fidx].used = TRUE;
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->field_file_map[fidx] = i;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->file_field_map[i] = fidx;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen /* update last_used if it's newer than ours */
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen if ((time_t)last_used[i] > cache->fields[fidx].field.last_used)
e2ce8d4a6ac5d82a906178148453e7613fab9ba0Timo Sirainen cache->fields[fidx].field.last_used = last_used[i];
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen dec = cache->fields[fidx].field.decision;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (cache->fields[fidx].field.last_used < max_drop_time &&
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->fields[fidx].field.last_used != 0 &&
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen (dec & MAIL_CACHE_DECISION_FORCED) == 0 &&
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen dec != MAIL_CACHE_DECISION_NO) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen /* time to drop this field. don't bother dropping
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen fields that have never been used. */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->need_compress_file_seq = cache->hdr->file_seq;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen names = p + 1;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return 0;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenstatic void copy_to_buf(struct mail_cache *cache, buffer_t *dest, bool add_new,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen size_t offset, size_t size)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen const void *data;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen unsigned int i, field;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen /* copy the existing fields */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen for (i = 0; i < cache->file_fields_count; i++) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen field = cache->file_field_map[i];
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen data = CONST_PTR_OFFSET(&cache->fields[field], offset);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append(dest, data, size);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (!add_new)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen /* copy newly wanted fields */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen for (i = 0; i < cache->fields_count; i++) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen data = CONST_PTR_OFFSET(&cache->fields[i], offset);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append(dest, data, size);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenstatic void copy_to_buf_byte(struct mail_cache *cache, buffer_t *dest,
2d2ebe91d56e9a158de000c9d0026f65600fbcfaTimo Sirainen bool add_new, size_t offset)
2d2ebe91d56e9a158de000c9d0026f65600fbcfaTimo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen const int *data;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen unsigned int i, field;
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen uint8_t byte;
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen /* copy the existing fields */
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen for (i = 0; i < cache->file_fields_count; i++) {
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen field = cache->file_field_map[i];
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen data = CONST_PTR_OFFSET(&cache->fields[field], offset);
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen byte = (uint8_t)*data;
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen buffer_append(dest, &byte, 1);
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen }
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen if (!add_new)
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen return;
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen /* copy newly wanted fields */
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen for (i = 0; i < cache->fields_count; i++) {
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen data = CONST_PTR_OFFSET(&cache->fields[i], offset);
d691782ae9eb4f6ab06bc42f1617e889e7c18a3bTimo Sirainen byte = (uint8_t)*data;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append(dest, &byte, 1);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenstatic int mail_cache_header_fields_update_locked(struct mail_cache *cache)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_t *buffer;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen uint32_t i, offset, dec_offset;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen int ret = 0;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (mail_cache_header_fields_read(cache) < 0 ||
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen mail_cache_header_fields_get_offset(cache, &offset, NULL) < 0)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return -1;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer = t_buffer_create(256);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen copy_to_buf(cache, buffer, FALSE,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen offsetof(struct mail_cache_field, last_used),
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen sizeof(uint32_t));
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen ret = mail_cache_write(cache, buffer->data, buffer->used,
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen offset + MAIL_CACHE_FIELD_LAST_USED());
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (ret == 0) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_set_used_size(buffer, 0);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen copy_to_buf_byte(cache, buffer, FALSE,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen offsetof(struct mail_cache_field, decision));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen dec_offset = offset +
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen MAIL_CACHE_FIELD_DECISION(cache->file_fields_count);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen ret = mail_cache_write(cache, buffer->data, buffer->used,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen dec_offset);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (ret == 0) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen for (i = 0; i < cache->file_fields_count; i++)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->fields[i].decision_dirty = FALSE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (ret == 0)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen cache->field_header_write_pending = FALSE;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return ret;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenint mail_cache_header_fields_update(struct mail_cache *cache)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
d78be924ad150840e018eda6a8a7d5e46a39bda2Timo Sirainen int ret;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (cache->locked) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen T_BEGIN {
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen ret = mail_cache_header_fields_update_locked(cache);
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen } T_END;
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen return ret;
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen }
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen if (mail_cache_lock(cache) <= 0)
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen return -1;
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen T_BEGIN {
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen ret = mail_cache_header_fields_update_locked(cache);
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen } T_END;
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen if (mail_cache_unlock(cache) < 0)
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen ret = -1;
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen return ret;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenvoid mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen struct mail_cache_header_fields hdr;
d78be924ad150840e018eda6a8a7d5e46a39bda2Timo Sirainen unsigned int field;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen const char *name;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen uint32_t i;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen i_zero(&hdr);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen hdr.fields_count = cache->file_fields_count;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen for (i = 0; i < cache->fields_count; i++) {
46631c1d903c409444b1b1c4a1d41a033c09ee37Timo Sirainen if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i))
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen hdr.fields_count++;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append(dest, &hdr, sizeof(hdr));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen /* we have to keep the field order for the existing fields. */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen copy_to_buf(cache, dest, TRUE,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen offsetof(struct mail_cache_field, last_used),
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen sizeof(uint32_t));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen copy_to_buf(cache, dest, TRUE,
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen offsetof(struct mail_cache_field, field_size),
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen sizeof(uint32_t));
8b247780e911909a9fdc47f69ce6d1478902ad98Timo Sirainen copy_to_buf_byte(cache, dest, TRUE,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen offsetof(struct mail_cache_field, type));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen copy_to_buf_byte(cache, dest, TRUE,
8b2cf1c1bd8ddcea0525b62fd35ba76e136828a1Timo Sirainen offsetof(struct mail_cache_field, decision));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen i_assert(dest->used == sizeof(hdr) +
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen (sizeof(uint32_t)*2 + 2) * hdr.fields_count);
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen /* add existing fields' names */
f28583935a4509d3f7932738bba6c548ef455c82Timo Sirainen for (i = 0; i < cache->file_fields_count; i++) {
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen field = cache->file_field_map[i];
e2a88d59c0d47d63ce1ad5b1fd95e487124a3fd4Timo Sirainen name = cache->fields[field].field.name;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append(dest, name, strlen(name)+1);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen /* add newly wanted fields' names */
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen for (i = 0; i < cache->fields_count; i++) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen name = cache->fields[i].field.name;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append(dest, name, strlen(name)+1);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen hdr.size = dest->used;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_write(dest, 0, &hdr, sizeof(hdr));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if ((hdr.size & 3) != 0)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen buffer_append_zero(dest, 4 - (hdr.size & 3));
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainenint mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen uint32_t *offset_r)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen{
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (mail_cache_header_fields_get_offset(cache, offset_r, NULL) < 0)
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return -1;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen if (*offset_r == 0) {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen *offset_r = offsetof(struct mail_cache_header,
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen field_header_offset);
981139bb2e446bb2050c1158614725f8413fd709Timo Sirainen } else {
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen *offset_r += offsetof(struct mail_cache_header_fields,
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen next_offset);
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen }
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen return 0;
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen}
e3678f7bfba87b5aa1446a1a7b9928272b4f72a3Timo Sirainen