mail-cache-fields.c revision a04b6515c20b431294626400e173d81f3d25889b
89a126810703c666309310d0f3189e9834d70b5bTimo Sirainen/* Copyright (c) 2004-2007 Dovecot authors, see the included COPYING file */
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen#define CACHE_FIELD_IS_NEWLY_WANTED(cache, field_idx) \
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen ((cache)->field_file_map[field_idx] == (uint32_t)-1 && \
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainenstatic bool field_has_fixed_size(enum mail_cache_field_type type)
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainenstatic bool field_decision_is_valid(enum mail_cache_decision_type type)
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainenstatic int field_type_verify(struct mail_cache *cache, unsigned int idx,
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen enum mail_cache_field_type type, unsigned int size)
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen const struct mail_cache_field *field = &cache->fields[idx].field;
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen "registered field %s type changed", field->name);
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen if (field->field_size != size && field_has_fixed_size(type)) {
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen "registered field %s size changed", field->name);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenvoid mail_cache_register_fields(struct mail_cache *cache,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen unsigned int new_idx;
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen unsigned int i, j;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < fields_count; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (hash_lookup_full(cache->field_name_hash, fields[i].name,
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen i_assert(fields[i].type < MAIL_CACHE_FIELD_COUNT);
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen /* check if the same header is being registered in the
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen same field array */
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen for (j = 0; j < i; j++) {
be889d9b142fbb5604a922c6955bd7f6ea32f163Timo Sirainen if (strcasecmp(fields[i].name, fields[j].name) == 0) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* @UNSAFE */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen cache->fields_count * sizeof(*cache->field_file_map),
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < fields_count; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* new index - save it */
8d6cb44a0161d88743756733f83c4fb278485987Timo Sirainen name = p_strdup(cache->field_pool, fields[i].name);
9bf2dc275ec21bff3d468ab1bc4fddc8874f7d1bTimo Sirainen if (!field_has_fixed_size(cache->fields[idx].field.type))
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen cache->fields[idx].field.field_size = (unsigned int)-1;
8d6cb44a0161d88743756733f83c4fb278485987Timo Sirainen hash_insert(cache->field_name_hash, name, POINTER_CAST(idx));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenmail_cache_register_lookup(struct mail_cache *cache, const char *name)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (hash_lookup_full(cache->field_name_hash, name,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return POINTER_CAST_TO(orig_value, unsigned int);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen return (unsigned int)-1;
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainenmail_cache_register_get_list(struct mail_cache *cache, pool_t pool,
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen unsigned int *count_r)
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen unsigned int i;
16598a1fb870ae40d6177755a4306216e4d6a4cdTimo Sirainen list = p_new(pool, struct mail_cache_field, cache->fields_count);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenstatic int mail_cache_header_fields_get_offset(struct mail_cache *cache,
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const struct mail_cache_header_fields *field_hdr;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen struct mail_cache_header_fields tmp_field_hdr;
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen unsigned int next_count = 0;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* find the latest header */
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen next_offset = cache->last_field_header_offset != 0 ?
44ff75ca53188056ff5a3e50428e3f2078800b3cTimo Sirainen mail_index_offset_to_uint32(cache->hdr->field_header_offset);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen while (next_offset != 0) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen "next_offset in field header loops");
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen sizeof(*field_hdr)) < 0)
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen field_hdr = CONST_PTR_OFFSET(cache->data, offset);
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen /* if we need to follow multiple offsets to get to
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen the last one, it's faster to just pread() the file
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen instead of going through cache */
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen mail_cache_set_syscall_error(cache, "pread()");
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen "next_offset points outside file");
44ff75ca53188056ff5a3e50428e3f2078800b3cTimo Sirainen mail_index_offset_to_uint32(field_hdr->next_offset);
6e235046e1d8e9d89fc948f5c623676c20421a28Timo Sirainen mail_cache_set_corrupted(cache, "missing header fields");
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT)
abb83d133dd082527d500916fca66a72fbbbaa8dTimo Sirainen cache->need_compress_file_seq = cache->hdr->file_seq;
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen if (cache->file_cache != NULL && invalidate) {
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen /* if this isn't the first header in file and we hadn't
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen read this before, we can't trust that the cached
e4423c16a4f798ecf75ca2b0c3ef834000faed4bTimo Sirainen data is valid */
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen file_cache_invalidate(cache->file_cache, offset,
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (mail_cache_map(cache, offset, field_hdr->size) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenint mail_cache_header_fields_read(struct mail_cache *cache)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen const struct mail_cache_header_fields *field_hdr = NULL;
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (mail_cache_header_fields_get_offset(cache, &offset, TRUE) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* no fields - the file is empty */
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen field_hdr = CONST_PTR_OFFSET(cache->data, offset);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen if (offset + field_hdr->size > cache->mmap_length) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "field header points outside file");
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* check the fixed size of the header. name[] has to be checked
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen separately */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen field_hdr->fields_count * (sizeof(uint32_t)*2 + 1 + 2)) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen mail_cache_set_corrupted(cache, "invalid field header size");
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen field_hdr = CONST_PTR_OFFSET(cache->data, offset);
265e9d8457aa5ce01b7aad954632aed8c36deeb8Timo Sirainen sizeof(unsigned int),
11768c622a8e5aaf0e5d9b3c9a872867b62b4830Timo Sirainen new_fields_count * sizeof(unsigned int));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED());
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count));
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* clear the old mapping */
6ebcdea168735ee76e32b871c1f50f3526690447Timo Sirainen max_drop_time = cache->index->map->hdr.day_stamp == 0 ? 0 :
6ebcdea168735ee76e32b871c1f50f3526690447Timo Sirainen cache->index->map->hdr.day_stamp - MAIL_CACHE_FIELD_DROP_SECS;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < field_hdr->fields_count; i++) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (p = names; p != end && *p != '\0'; p++) ;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen "field header names corrupted");
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen mail_cache_set_corrupted(cache, "field type corrupted");
5c0034beb9933bca2a8b7d83d11dface1ea3b7faTimo Sirainen "field decision type corrupted");
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (hash_lookup_full(cache->field_name_hash, names,
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen /* already exists, see if decision can be updated */
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen fidx = POINTER_CAST_TO(orig_value, unsigned int);
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen if (cache->field_file_map[fidx] != (uint32_t)-1) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen /* update last_used if it's newer than ours */
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen if (last_used[i] > cache->fields[fidx].last_used)
a04b6515c20b431294626400e173d81f3d25889bTimo Sirainen if ((time_t)cache->fields[fidx].last_used < max_drop_time &&
9c4c535b86e9473ad97c6e9242ed84f3d9d69d0dTimo Sirainen /* time to drop this field. don't bother dropping
9c4c535b86e9473ad97c6e9242ed84f3d9d69d0dTimo Sirainen fields that have never been used. */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen cache->need_compress_file_seq = cache->hdr->file_seq;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainenstatic void copy_to_buf(struct mail_cache *cache, buffer_t *dest, bool add_new,
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen unsigned int i, field;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* copy the existing fields */
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen for (i = 0; i < cache->file_fields_count; i++) {
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen data = CONST_PTR_OFFSET(&cache->fields[field], offset);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* copy newly wanted fields */
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen data = CONST_PTR_OFFSET(&cache->fields[i], offset);
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainenstatic void copy_to_buf_byte(struct mail_cache *cache, buffer_t *dest,
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen unsigned int i, field;
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* copy the existing fields */
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen for (i = 0; i < cache->file_fields_count; i++) {
e96fb85799dc95603bb1a6b4d3685df2d042a2f8Timo Sirainen data = CONST_PTR_OFFSET(&cache->fields[field], offset);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* copy newly wanted fields */
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen data = CONST_PTR_OFFSET(&cache->fields[i], offset);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenstatic int mail_cache_header_fields_update_locked(struct mail_cache *cache)
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen if (mail_cache_header_fields_read(cache) < 0 ||
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen mail_cache_header_fields_get_offset(cache, &offset, FALSE) < 0)
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen buffer = buffer_create_dynamic(pool_datastack_create(), 256);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen offsetof(struct mail_cache_field_private, last_used),
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen ret = mail_cache_write(cache, buffer->data, buffer->used,
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen MAIL_CACHE_FIELD_DECISION(cache->file_fields_count);
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen ret = mail_cache_write(cache, buffer->data, buffer->used,
2c20ffcb5bb1ccdfdcd0b0ff0c7296f65b990362Timo Sirainen for (i = 0; i < cache->file_fields_count; i++)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenint mail_cache_header_fields_update(struct mail_cache *cache)
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen ret = mail_cache_header_fields_update_locked(cache);
eddd9bf1a1369aea4a2715f6be1137da6d17d293Timo Sirainen ret = mail_cache_header_fields_update_locked(cache);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenvoid mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen unsigned int field;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen /* return newly added fields' last_used as
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen the current time */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen /* we have to keep the field order for the existing fields. */
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen offsetof(struct mail_cache_field_private, last_used),
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen offsetof(struct mail_cache_field, field_size),
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* add existing fields' names */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen for (i = 0; i < cache->file_fields_count; i++) {
8c02331f9f569d8b30e74b6bc8550734d65f9daeTimo Sirainen /* add newly wanted fields' names */
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainenint mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
8b681dae1e8fa564649e703ab17398dcfaf896e4Timo Sirainen if (mail_cache_header_fields_get_offset(cache, offset_r, FALSE) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen *offset_r = offsetof(struct mail_cache_header,