mail-cache-lookup.c revision b1f58fea9e1813b48735a021043f1cea2b495c3b
b355dcb54194f498921743ca33304eac35d89718Stephen Gallagher/* Copyright (C) 2003-2004 Timo Sirainen */
7a14e8f66c0e932fe2954d792614a3b61d444bd1Jakub Hrozekint mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
c938f4ba417328fe62eded0806b2d9ca053f34a5Stephen Gallagher const struct mail_cache_record *cache_rec;
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek if (offset + sizeof(*cache_rec) > cache->mmap_length) {
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek mail_cache_set_corrupted(cache, "record points outside file");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan mail_cache_set_corrupted(cache, "invalid record size");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (mail_cache_map(cache, offset, cache_rec->size) < 0)
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher if (cache_rec->size > cache->mmap_length ||
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher offset + cache_rec->size > cache->mmap_length) {
2ea6196484055397cc4bc011c5960f790431fa9dStephen Gallagher mail_cache_set_corrupted(cache, "record points outside file");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivanmail_cache_lookup_offset(struct mail_cache *cache, struct mail_index_view *view,
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher if (mail_index_lookup_ext_full(view, seq, cache->ext_id,
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek if (!mail_index_map_get_ext_idx(map, cache->ext_id, &idx)) {
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek /* no cache */
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan for (i = 0; i < 2; i++) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (cache->hdr->file_seq == ext->reset_id) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivanmail_cache_foreach_rec(struct mail_cache_view *view, uint32_t *offset,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan mail_cache_foreach_callback_t *callback, void *context)
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan unsigned int field;
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (mail_cache_get_record(view->cache, *offset, &cache_rec) < 0)
c938f4ba417328fe62eded0806b2d9ca053f34a5Stephen Gallagher if (max_size < sizeof(*cache_rec) + sizeof(uint32_t)*2) {
c938f4ba417328fe62eded0806b2d9ca053f34a5Stephen Gallagher mail_cache_set_corrupted(cache, "record has invalid size");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan for (pos = sizeof(*cache_rec); pos < max_size; ) {
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher *((const uint32_t *)CONST_PTR_OFFSET(cache_rec, pos));
c938f4ba417328fe62eded0806b2d9ca053f34a5Stephen Gallagher if (file_field >= cache->file_fields_count) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan /* new field, have to re-read fields header to figure
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher out its size */
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (mail_cache_header_fields_read(cache) < 0)
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher if (file_field >= cache->file_fields_count) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan "field index too large (%u >= %u)",
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan /* field reading might have re-mmaped the file and
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan caused cache_rec to break. need to get it again. */
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (mail_cache_get_record(view->cache, *offset,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan data_size = cache->fields[field].field.field_size;
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher if (data_size > cache_rec->size || next_pos > cache_rec->size) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan "record continues outside its allocated size");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan ret = callback(view, field, CONST_PTR_OFFSET(cache_rec, pos),
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivanstatic bool find_offset(struct mail_cache_view *view, uint32_t offset)
c938f4ba417328fe62eded0806b2d9ca053f34a5Stephen Gallagher unsigned int i, count;
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan offsets = array_get(&view->tmp_offsets, &count);
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher for (i = 0; i < count; i++) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivanint mail_cache_foreach(struct mail_cache_view *view, uint32_t seq,
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan mail_cache_foreach_callback_t *callback, void *context)
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher (void)mail_cache_open_and_verify(view->cache);
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if ((ret = mail_cache_lookup_offset(view->cache, view->view,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan "record list is circular");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan array_append(&view->tmp_offsets, &offset, 1);
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher ret = mail_cache_foreach_rec(view, &offset,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (ret > 0 && view->trans_seq1 <= seq && view->trans_seq2 >= seq &&
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan mail_cache_lookup_offset(view->cache, view->trans_view,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan "record list is circular");
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan array_append(&view->tmp_offsets, &offset, 1);
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallaghermail_cache_seq_callback(struct mail_cache_view *view, uint32_t field,
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher buffer_write(view->cached_exists_buf, field,
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagherstatic int mail_cache_seq(struct mail_cache_view *view, uint32_t seq)
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher /* wrapped, we'll have to clear the buffer */
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher memset(buffer_get_modifiable_data(view->cached_exists_buf,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan ret = mail_cache_foreach(view, seq, mail_cache_seq_callback, NULL);
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivanint mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan unsigned int field)
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan i_assert(field < view->cache->fields_count);
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher (void)mail_cache_open_and_verify(view->cache);
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher file_field = view->cache->field_file_map[field];
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan data = buffer_get_data(view->cached_exists_buf, &size);
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallaghermail_cache_field_get_decision(struct mail_cache *cache, unsigned int field)
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivanmail_cache_lookup_callback(struct mail_cache_view *view __attr_unused__,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan struct mail_cache_lookup_context *ctx = context;
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan buffer_append(ctx->dest_buf, data, data_size);
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivanmail_cache_lookup_bitmask_callback(struct mail_cache_view *view __attr_unused__,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan struct mail_cache_lookup_context *ctx = context;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher unsigned char *dest;
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher /* merge all bits */
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan dest = buffer_get_space_unsafe(ctx->dest_buf, 0, data_size);
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher for (i = 0; i < data_size; i++)
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher dest[i] |= ((const unsigned char *)data)[i];
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagherint mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
7797e361155f7ce937085fd98e360469d7baf1b6Jakub Hrozek if ((ret = mail_cache_field_exists(view, seq, field)) <= 0)
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan /* should exist. find it. */
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (view->cache->fields[field].field.type != MAIL_CACHE_FIELD_BITMASK) {
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher ret = mail_cache_foreach(view, seq, mail_cache_lookup_callback,
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan /* make sure we're cleared first */
7797e361155f7ce937085fd98e360469d7baf1b6Jakub Hrozek data_size = view->cache->fields[field].field.field_size;
7797e361155f7ce937085fd98e360469d7baf1b6Jakub Hrozek memset(buffer_get_space_unsafe(dest_buf, 0, data_size),
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan unsigned int *fields;
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagherheaders_find_callback(struct mail_cache_view *view, uint32_t field,
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher const void *data, size_t data_size, void *context)
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan struct header_lookup_context *ctx = context;
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan struct header_lookup_data_rec *hdr_data_rec;
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan unsigned int i, lines_count;
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan if (field > ctx->max_field || ctx->fields_found[field] != 1) {
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher /* a) don't want it, b) duplicate */
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan /* data = { line_nums[], 0, "headers" } */
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan for (i = 0; data_size >= sizeof(uint32_t); i++) {
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan hdr_data_rec = t_new(struct header_lookup_data_rec, 1);
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan hdr_data_rec->offset = (const char *)&lines[lines_count+1] -
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan hdr_data_rec->data_size = (uint32_t)data_size;
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan for (i = 0; i < lines_count; i++) {
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher buffer_append(ctx->data, &hdr_data, sizeof(hdr_data));
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherstatic int header_lookup_data_cmp(const void *p1, const void *p2)
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagher const struct header_lookup_data *d1 = p1, *d2 = p2;
b355dcb54194f498921743ca33304eac35d89718Stephen Gallagher return (int)d1->line_num - (int)d2->line_num;
52261fe16203dec6e6f69177c6d0a810b47d073fStephen Gallagherint mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest,
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher ctx.fields = t_new(unsigned int, fields_count);
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher buf = buffer_create_dynamic(pool_datastack_create(), 32);
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher for (i = 0; i < fields_count; i++) {
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher i_assert(fields[i] < cache->fields_count);
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher if (cache->field_file_map[fields[i]] == (unsigned int)-1) {
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher /* not cached at all */
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher ctx.fields_found = buffer_get_modifiable_data(buf, NULL);
65a9065538fd85e6ead925d344e6b421900eb8c2Jakub Hrozek ctx.data = buffer_create_dynamic(pool_datastack_create(), 256);
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher /* we need to return them in sorted order. create array:
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher { line number -> cache file offset } */
e59e09b5010f262228bbdeb92a79b733bf5854b3Stephen Gallagher ret = mail_cache_foreach(view, seq, headers_find_callback, &ctx);
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan /* check that all fields were found */
f9fdc87c80f2744780c6a0f2bf5b1b57bcbb095aYuri Chornoivan for (i = 0; i < fields_count; i++)
c938f4ba417328fe62eded0806b2d9ca053f34a5Stephen Gallagher mail_cache_decision_lookup(view, seq, fields[i]);
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher data = buffer_get_modifiable_data(ctx.data, &size);
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan qsort(data, size, sizeof(*data), header_lookup_data_cmp);
bde69429374859acff41273c0771d2b5f5c199b1Yuri Chornoivan /* then start filling dest buffer from the headers */
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher for (i = 0; i < size; i++) {
dd3ba5c5b7d2a9d109963ae9e6c94fff34872221Stephen Gallagher start = CONST_PTR_OFFSET(cache->data, data[i].data->offset);
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher if (*p == '\n' &&
cbe7c54c2caf718bdea7ca6660ba8193d759d2d5Stephen Gallagher (p+1 == end || (p[1] != ' ' && p[1] != '\t'))) {