squat-uidlist.c revision 4fbd8de028e97cb14496fbcf74f3b878831274ae
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (C) 2006 Timo Sirainen */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
08d6658a4e2ec8104cd1307f6baa75fdb07a24f8Mark Washenberger#include "lib.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "array.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "ostream.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "file-cache.h"
08d6658a4e2ec8104cd1307f6baa75fdb07a24f8Mark Washenberger#include "mmap-util.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "read-full.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "write-full.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "squat-trie.h"
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include "squat-uidlist.h"
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include <stdio.h>
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include <unistd.h>
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include <fcntl.h>
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#include <sys/stat.h>
25d624dd86700c82cd28427f3d3bebe7c8f7f459Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#define UIDLIST_COMPRESS_PERCENTAGE 30
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#define UIDLIST_UID_COMPRESS_PERCENTAGE 20
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#define UIDLIST_COMPRESS_MIN_SIZE (1024*8)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#define SQUAT_UIDLIST_FLUSH_THRESHOLD (1024*1024*31)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#define UID_NODE_PREV_FLAG_OLD 0x00000001
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen#define UID_LIST_IDX_FLAG_SINGLE 0x80000000
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstruct squat_uidlist_header {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t uidvalidity;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t used_file_size;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t deleted_space;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t uid_max;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t uid_count;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint8_t uids_expunged; /* updated without locking */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint8_t unused[3];
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t node_count;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen};
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstruct uid_node {
04870054863757edf048c81dcce3c5e7dec453cdTimo Sirainen struct uid_node *prev;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen};
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstruct squat_uidlist_get_context {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct squat_uidlist *uidlist;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ARRAY_TYPE(seq_range) *result;
bd63b5b860658b01b1f46f26d406e1e4a9dc019aTimo Sirainen
15a07b47846c47a81d69a14d649564e222d6f742Timo Sirainen uint32_t filter_uid_pos;
cd2cd224d3216a243d55c71c298a5b7684de0ac4Timo Sirainen};
c1faff067b29fb48426cb84260adba563e93189aTimo Sirainen
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainenstruct squat_uidlist {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct squat_trie *trie;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t uidvalidity;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen char *filepath;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen int fd;
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen struct ostream *output;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen dev_t dev;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen ino_t ino;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen void *mmap_base;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen const uint8_t *const_mmap_base;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen size_t mmap_size;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen struct file_cache *file_cache;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct squat_uidlist_header hdr;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ARRAY_DEFINE(lists, struct uid_node);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t first_new_list_idx;
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen uint32_t current_uid;
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen pool_t node_pool;
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen size_t node_pool_used;
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen buffer_t *tmp_buf, *list_buf;
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen unsigned int check_expunges:1;
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen unsigned int write_failed:1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen unsigned int mmap_disable:1;
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen};
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainen
eacce2276278ce6a8176a9a100807dba50bbfb36Timo Sirainenstruct squat_uidlist_compress_ctx {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct squat_uidlist *uidlist;
e1203014de25c8c3d3975a9f4b4a04616df4bba2Timo Sirainen const ARRAY_TYPE(seq_range) *existing_uids;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct ostream *output;
e1203014de25c8c3d3975a9f4b4a04616df4bba2Timo Sirainen char *tmp_path;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen pool_t node_pool;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct uid_node *last_node;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ARRAY_TYPE(seq_range) seen_uids;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct squat_uidlist_header hdr;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen unsigned int seen_expunged:1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen unsigned int failed:1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen};
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstatic void squat_uidlist_close(struct squat_uidlist *uidlist);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstatic void
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainensquat_uidlist_set_syscall_error(struct squat_uidlist *uidlist,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen const char *function)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen i_error("%s failed with index search uidlist file %s: %m",
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen function, uidlist->filepath);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
411f318ed3a25fa66c1b932e10df43841e2725c9Timo Sirainenstatic int squat_uidlist_check_header(struct squat_uidlist *uidlist,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen const struct squat_uidlist_header *hdr,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uoff_t file_size)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (hdr->used_file_size == 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* crashed before writing was finished */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (hdr->uidvalidity != uidlist->uidvalidity) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen squat_trie_set_corrupted(uidlist->trie,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen "uidlist: uidvalidity changed");
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (hdr->used_file_size > file_size) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen squat_trie_set_corrupted(uidlist->trie,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen "uidlist: used_file_size too large");
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
411f318ed3a25fa66c1b932e10df43841e2725c9Timo Sirainen }
411f318ed3a25fa66c1b932e10df43841e2725c9Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstatic int squat_uidlist_read_header(struct squat_uidlist *uidlist)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen int ret;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ret = pread_full(uidlist->fd, &uidlist->hdr, sizeof(uidlist->hdr), 0);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (ret < 0)
3a9eb305fd4aad5502cb7e64625874385ab5bc19Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "pread_full()");
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen return ret;
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstatic int squat_uidlist_map(struct squat_uidlist *uidlist)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct stat st;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen int ret;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (!uidlist->mmap_disable) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen const struct squat_uidlist_header *hdr = uidlist->mmap_base;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (hdr != NULL && hdr->used_file_size <= uidlist->mmap_size) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* everything is already mapped */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->hdr = *hdr;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen } else {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if ((ret = squat_uidlist_read_header(uidlist)) < 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (ret == 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (fstat(uidlist->fd, &st) < 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "fstat()");
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->dev = st.st_dev;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->ino = st.st_ino;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (st.st_size <= (off_t)sizeof(uidlist->hdr))
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (uidlist->mmap_base != NULL) {
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "munmap()");
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen }
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->const_mmap_base = NULL;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (!uidlist->mmap_disable) {
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->mmap_size = st.st_size;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->mmap_base =
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen mmap(NULL, uidlist->mmap_size, PROT_READ | PROT_WRITE,
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen MAP_SHARED, uidlist->fd, 0);
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (uidlist->mmap_base == MAP_FAILED) {
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->mmap_size = 0;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->mmap_base = NULL;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "mmap()");
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen return -1;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen }
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->const_mmap_base = uidlist->mmap_base;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr));
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen } else {
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen /* the header is always read separately. everything between it
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen and the used_file_size doesn't change */
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen file_cache_invalidate(uidlist->file_cache,
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->hdr.used_file_size, (uoff_t)-1);
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen }
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (squat_uidlist_check_header(uidlist, &uidlist->hdr, st.st_size) < 0)
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen return 0;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (uidlist->hdr.uids_expunged)
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->check_expunges = TRUE;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uidlist->first_new_list_idx = uidlist->hdr.used_file_size;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen return 1;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen}
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainenstatic int squat_uidlist_open(struct squat_uidlist *uidlist)
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen int ret;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen i_assert(uidlist->fd == -1);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen uidlist->fd = open(uidlist->filepath, O_RDWR, 0600);
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen if (uidlist->fd == -1) {
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if (errno == ENOENT)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "open()");
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen return -1;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen }
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uidlist->mmap_disable)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->file_cache = file_cache_new(uidlist->fd);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen if ((ret = squat_uidlist_map(uidlist)) == 0) {
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen /* broken */
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen if (unlink(uidlist->filepath) < 0)
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "unlink()");
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen squat_uidlist_close(uidlist);
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen }
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen return ret;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen}
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainenstatic int squat_uidlist_create(struct squat_uidlist *uidlist)
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainen{
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainen i_assert(uidlist->fd == -1);
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen /* we should get here only if normal file opening failed */
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen uidlist->fd = open(uidlist->filepath, O_RDWR | O_CREAT | O_TRUNC, 0600);
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen if (uidlist->fd == -1) {
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "open()");
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen return -1;
34015eb0b74735f2fac07c12697bde20a94735e6Timo Sirainen }
34015eb0b74735f2fac07c12697bde20a94735e6Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uidlist->mmap_disable)
34015eb0b74735f2fac07c12697bde20a94735e6Timo Sirainen uidlist->file_cache = file_cache_new(uidlist->fd);
34015eb0b74735f2fac07c12697bde20a94735e6Timo Sirainen return 0;
34015eb0b74735f2fac07c12697bde20a94735e6Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
34015eb0b74735f2fac07c12697bde20a94735e6Timo Sirainenstatic void squat_uidlist_close(struct squat_uidlist *uidlist)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uidlist->file_cache != NULL)
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen file_cache_free(&uidlist->file_cache);
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainen if (uidlist->mmap_base != NULL) {
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "munmap()");
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen uidlist->mmap_base = NULL;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen }
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen uidlist->const_mmap_base = NULL;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen uidlist->mmap_size = 0;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen if (uidlist->fd != -1) {
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen if (close(uidlist->fd) < 0)
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "close()");
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen uidlist->fd = -1;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen }
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen}
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstruct squat_uidlist *
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainensquat_uidlist_init(struct squat_trie *trie, const char *path,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t uidvalidity, bool mmap_disable)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct squat_uidlist *uidlist;
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainen uidlist = i_new(struct squat_uidlist, 1);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->trie = trie;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->filepath = i_strdup(path);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->uidvalidity = uidvalidity;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->fd = -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->first_new_list_idx = 1;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen uidlist->mmap_disable = mmap_disable;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen i_array_init(&uidlist->lists, 65536);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->node_pool =
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen pool_alloconly_create(MEMPOOL_GROWING"squat uidlist node pool",
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen 65536);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->tmp_buf = buffer_create_dynamic(default_pool, 16);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->list_buf = buffer_create_dynamic(default_pool, 256);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return uidlist;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenvoid squat_uidlist_deinit(struct squat_uidlist *uidlist)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen{
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen squat_uidlist_close(uidlist);
0b2f7be9fadfd4026a9174e51170890cde3edf48Timo Sirainen
45c872f65e4f327ef166c6e2b71bb43e188ac562Timo Sirainen pool_unref(uidlist->node_pool);
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen array_free(&uidlist->lists);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen buffer_free(uidlist->tmp_buf);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen buffer_free(uidlist->list_buf);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen i_free(uidlist->filepath);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen i_free(uidlist);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainenint squat_uidlist_refresh(struct squat_uidlist *uidlist)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen struct stat st;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen int ret;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uidlist->fd != -1) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (stat(uidlist->filepath, &st) < 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (errno == ENOENT)
ec5fec7eab19e134a2607b7e224b3e14a1771ee0Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen squat_uidlist_set_syscall_error(uidlist, "stat()");
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen }
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen if (st.st_ino == uidlist->ino &&
ec5fec7eab19e134a2607b7e224b3e14a1771ee0Timo Sirainen CMP_DEV_T(st.st_dev, uidlist->dev)) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* no need to reopen, just remap */
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if ((ret = squat_uidlist_map(uidlist)) != 0)
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen return ret < 0 ? -1 : 0;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen /* broken file */
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen }
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen squat_uidlist_close(uidlist);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (squat_uidlist_open(uidlist) < 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return 0;
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenint squat_uidlist_get_last_uid(struct squat_uidlist *uidlist, uint32_t *uid_r)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen *uid_r = uidlist->hdr.uid_max;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
04870054863757edf048c81dcce3c5e7dec453cdTimo Sirainen
04870054863757edf048c81dcce3c5e7dec453cdTimo Sirainenint squat_uidlist_add(struct squat_uidlist *uidlist, uint32_t *_uid_list_idx,
04870054863757edf048c81dcce3c5e7dec453cdTimo Sirainen uint32_t uid)
04870054863757edf048c81dcce3c5e7dec453cdTimo Sirainen{
e1203014de25c8c3d3975a9f4b4a04616df4bba2Timo Sirainen uint32_t uid_list_idx = *_uid_list_idx;
e1203014de25c8c3d3975a9f4b4a04616df4bba2Timo Sirainen struct uid_node *node, *old_node;
e593e507ee5ea3869271a631874c5c4b5c7a294dTimo Sirainen
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen i_assert(uid > uidlist->hdr.uid_max || uid == uidlist->current_uid);
e1203014de25c8c3d3975a9f4b4a04616df4bba2Timo Sirainen
e1203014de25c8c3d3975a9f4b4a04616df4bba2Timo Sirainen if (uid_list_idx == 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen *_uid_list_idx = uid | UID_LIST_IDX_FLAG_SINGLE;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
ec5fec7eab19e134a2607b7e224b3e14a1771ee0Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uid > uidlist->hdr.uid_max) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->current_uid = uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->hdr.uid_max = uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->hdr.uid_count++;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uid_list_idx < uidlist->first_new_list_idx) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* continue an existing list in the uidlist file */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen old_node = POINTER_CAST((uid_list_idx << 1) |
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen UID_NODE_PREV_FLAG_OLD);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uid_list_idx = uidlist->first_new_list_idx +
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen array_count(&uidlist->lists);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen node = array_append_space(&uidlist->lists);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->hdr.node_count++;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen } else if ((uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen uint32_t old_uid = uid_list_idx & ~UID_LIST_IDX_FLAG_SINGLE;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uid == old_uid) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* trying to add the same uid again */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* convert single UID to a list */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->node_pool_used += sizeof(struct uid_node);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen old_node = p_new(uidlist->node_pool, struct uid_node, 1);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen old_node->uid = old_uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uid_list_idx = uidlist->first_new_list_idx +
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen array_count(&uidlist->lists);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen node = array_append_space(&uidlist->lists);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen uidlist->hdr.node_count++;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen } else {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* update an in-memory list */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t arr_idx = uid_list_idx - uidlist->first_new_list_idx;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (arr_idx >= array_count(&uidlist->lists)) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* broken */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen squat_trie_set_corrupted(uidlist->trie,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen "corrupted uidlist index (adding)");
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen return -1;
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen }
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen node = array_idx_modifiable(&uidlist->lists, arr_idx);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (node->uid == uid) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* trying to add the same uid again */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->node_pool_used += sizeof(struct uid_node);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen old_node = p_new(uidlist->node_pool, struct uid_node, 1);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen *old_node = *node;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen node->prev = old_node;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen node->uid = uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen *_uid_list_idx = uid_list_idx;
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen return 0;
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen}
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainenstatic int
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainensquat_uidlist_map_area(struct squat_uidlist *uidlist,
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen size_t offset, size_t size)
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen{
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen ssize_t ret;
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen if (uidlist->file_cache == NULL)
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen return 0;
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen ret = file_cache_read(uidlist->file_cache, offset, size);
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen if (ret < 0) {
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen squat_uidlist_set_syscall_error(uidlist, "file_cache_read()");
e5b723864630e40c9028808ef417dd3d6fbf495bTimo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->const_mmap_base =
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen file_cache_get_map(uidlist->file_cache, &uidlist->mmap_size);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstatic int
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainensquat_uidlist_map_list(struct squat_uidlist *uidlist, size_t offset,
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen const uint8_t **data_r, uint32_t *size_r)
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen{
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen const uint8_t *data, *end;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen size_t data_offset;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen uint32_t size;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (squat_uidlist_map_area(uidlist, offset, 128) < 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (offset >= uidlist->mmap_size)
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen return -1;
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen data = uidlist->const_mmap_base + offset;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen end = uidlist->const_mmap_base + uidlist->mmap_size;
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen size = _squat_trie_unpack_num(&data, end);
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen data_offset = data - uidlist->const_mmap_base;
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen if (squat_uidlist_map_area(uidlist, data_offset, size) < 0)
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen return -1;
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen if (data_offset + size > uidlist->mmap_size)
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen return -1;
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen *data_r = uidlist->const_mmap_base + data_offset;
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen *size_r = size;
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen return 0;
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainenstatic int
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainensquat_uidlist_copy_existing(struct squat_uidlist *uidlist, size_t offset,
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen uint32_t *prev_uid_r, uint32_t *written_uid_r)
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen{
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen const uint8_t *data, *data_start, *end, *p = NULL;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t size, num, prev_uid, next_uid;
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (squat_uidlist_map_list(uidlist, offset, &data, &size) < 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen data_start = data;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen end = data + size;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen prev_uid = next_uid = _squat_trie_unpack_num(&data, end);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen p = data;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen while (data != end) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen num = _squat_trie_unpack_num(&data, end);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen next_uid = prev_uid + (num >> 1) + 1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if ((num & 1) != 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* prev_uid..next_uid */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (data == end) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* try to increase this range */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen break;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen }
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen /* beginning a new uid/range */
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen num = _squat_trie_unpack_num(&data, end);
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen next_uid += num + 1;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen prev_uid = next_uid;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen p = data;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen }
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen prev_uid = next_uid;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen p = data;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen }
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen *written_uid_r = prev_uid;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen *prev_uid_r = next_uid;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen uidlist->hdr.deleted_space +=
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen (end - (const uint8_t *)uidlist->const_mmap_base) - offset;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen buffer_append(uidlist->list_buf, data_start, p - data_start);
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen return 0;
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen}
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainenstatic int
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainensquat_uidlist_write_range(struct squat_uidlist *uidlist,
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen const struct uid_node *node,
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uint32_t *prev_uid_r, uint32_t *written_uid_r,
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen int level)
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen{
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen buffer_t *buffer = uidlist->list_buf;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen uint32_t written_uid, prev_uid;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen uint32_t prev_idx = POINTER_CAST_TO(node->prev, uint32_t);
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen *prev_uid_r = node->uid;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen if (node->prev == NULL) {
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen /* first UID */
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen _squat_trie_pack_num(buffer, node->uid);
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen } else {
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen if ((prev_idx & UID_NODE_PREV_FLAG_OLD) != 0) {
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen prev_idx >>= 1;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen if (squat_uidlist_copy_existing(uidlist, prev_idx,
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen &prev_uid,
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen &written_uid) < 0 ||
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen prev_uid >= node->uid) {
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen squat_trie_set_corrupted(uidlist->trie,
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen "corrupted continued uidlist index");
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen return -1;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen }
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen } else {
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen if (squat_uidlist_write_range(uidlist, node->prev,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen &prev_uid, &written_uid,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen level+1) < 0)
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen /* prev_uid contains the previous node's UID.
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen written_uid contains the last written UID. */
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen if (prev_uid + 1 == node->uid) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (level != 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* this node continue the range */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen *written_uid_r = written_uid;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen return 0;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen } else {
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen /* finishing range */
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen _squat_trie_pack_num(buffer, 1 |
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ((node->uid - written_uid - 1) << 1));
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen i_assert(prev_uid < node->uid);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (written_uid != prev_uid) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen i_assert(written_uid < prev_uid);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* range ends at prev_uid */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen _squat_trie_pack_num(buffer, 1 |
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ((prev_uid - written_uid - 1) << 1));
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* next uid/range */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen _squat_trie_pack_num(buffer, node->uid - prev_uid - 1);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen } else {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* no range */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen _squat_trie_pack_num(buffer,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ((node->uid - prev_uid - 1) << 1));
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen *written_uid_r = node->uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainenstatic int squat_uidlist_write_init(struct squat_uidlist *uidlist)
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen{
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen i_assert(uidlist->output == NULL);
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen if (uidlist->fd == -1) {
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen if (squat_uidlist_create(uidlist) < 0)
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen return -1;
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen }
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen uidlist->output = o_stream_create_file(uidlist->fd, default_pool,
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen 0, FALSE);
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr)) {
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen /* creating a new file, write a dummy header */
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen o_stream_seek(uidlist->output, 0);
c3d40f3092af25cad9e807a85eaad4d92aab107bTimo Sirainen o_stream_send(uidlist->output, &uidlist->hdr,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen sizeof(uidlist->hdr));
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen } else {
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen o_stream_seek(uidlist->output,
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen uidlist->hdr.used_file_size);
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen }
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen return 0;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen}
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainenstatic int squat_uidlist_write_listbuf(struct squat_uidlist *uidlist,
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen struct ostream *output)
411f318ed3a25fa66c1b932e10df43841e2725c9Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* write size + buffer */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen buffer_set_used_size(uidlist->tmp_buf, 0);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen _squat_trie_pack_num(uidlist->tmp_buf, uidlist->list_buf->used);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen if (o_stream_send(output, uidlist->tmp_buf->data,
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen uidlist->tmp_buf->used) < 0 ||
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen o_stream_send(output, uidlist->list_buf->data,
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen uidlist->list_buf->used) < 0) {
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen return -1;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen }
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen return 0;
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen}
bd4de9c8152e6ea032c1cb1df8b79635ff5ddf9eTimo Sirainen
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainenint squat_uidlist_finish_list(struct squat_uidlist *uidlist,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t *_uid_list_idx)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
4530cfa7456c10cd03fe9120c75f8bcb2f623ba4Timo Sirainen uint32_t uid_list_idx = *_uid_list_idx;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen const struct uid_node *node;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uint32_t prev_uid, written_uid;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen if ((uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* this is a single UID "list" */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uid_list_idx < uidlist->first_new_list_idx) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* the list hasn't changed */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uid_list_idx -= uidlist->first_new_list_idx;
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen if (uid_list_idx >= array_count(&uidlist->lists)) {
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen /* broken */
04a7b696e5255aa956277a0f7cabee736c69ec96Timo Sirainen squat_trie_set_corrupted(uidlist->trie,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen "corrupted uidlist index (finishing)");
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* write the uidlist into a buffer */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen node = array_idx(&uidlist->lists, uid_list_idx);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen buffer_set_used_size(uidlist->list_buf, 0);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (squat_uidlist_write_range(uidlist, node,
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen &prev_uid, &written_uid, 0) < 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->write_failed = TRUE;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen if (uidlist->output == NULL) {
714e2da5096fb52b8845d3c79f9bb26225a606c9Timo Sirainen if (squat_uidlist_write_init(uidlist) < 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->write_failed = TRUE;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen /* new uidlist index is the offset in uidlist file */
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen *_uid_list_idx = uidlist->output->offset;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (squat_uidlist_write_listbuf(uidlist, uidlist->output) < 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->write_failed = TRUE;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return 0;
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen}
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenstatic void squat_uidlist_write_header(struct squat_uidlist *uidlist)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen{
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen uidlist->hdr.used_file_size = uidlist->output->offset;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
cd2cd224d3216a243d55c71c298a5b7684de0ac4Timo Sirainen o_stream_seek(uidlist->output, 0);
cd2cd224d3216a243d55c71c298a5b7684de0ac4Timo Sirainen o_stream_send(uidlist->output, &uidlist->hdr, sizeof(uidlist->hdr));
6882df5fbca4a09cdaa95f54d70bb31b5920528cTimo Sirainen}
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainenint squat_uidlist_flush(struct squat_uidlist *uidlist, uint32_t uid_validity)
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen{
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen int ret = uidlist->write_failed ? -1 : 0;
cd2cd224d3216a243d55c71c298a5b7684de0ac4Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uidlist->output != NULL) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (ret == 0) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->hdr.uidvalidity = uid_validity;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen squat_uidlist_write_header(uidlist);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen o_stream_destroy(&uidlist->output);
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen }
cd2cd224d3216a243d55c71c298a5b7684de0ac4Timo Sirainen
ac0fed903142d28ae3a1d5d00d2097fdf161b138Timo Sirainen array_clear(&uidlist->lists);
ac0fed903142d28ae3a1d5d00d2097fdf161b138Timo Sirainen p_clear(uidlist->node_pool);
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->node_pool_used = 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->write_failed = FALSE;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen uidlist->current_uid = 0;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (uidlist->fd != -1) {
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen if (squat_uidlist_map(uidlist) <= 0)
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen ret = -1;
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen }
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainen return ret;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen}
b66d803de86bfb411165b3465b0d9ef64ecfe2a1Timo Sirainen
02a54da28f376dd66d7939d8546a196a0045b486Timo Sirainenbool squat_uidlist_need_compress(struct squat_uidlist *uidlist,
23fdad6c7e2581921f511e24cd9371c9eaebcef9Timo Sirainen unsigned int current_message_count)
23fdad6c7e2581921f511e24cd9371c9eaebcef9Timo Sirainen{
8552b0cad8ffe9ccb8270577ba28b8010c89af11Timo Sirainen uint32_t max_del_space, max_uid_del_count;
if (uidlist->hdr.used_file_size >= UIDLIST_COMPRESS_MIN_SIZE) {
/* see if we've reached the max. deleted space in file */
max_del_space = uidlist->hdr.used_file_size / 100 *
UIDLIST_COMPRESS_PERCENTAGE;
if (uidlist->hdr.deleted_space > max_del_space)
return TRUE;
}
if (uidlist->hdr.uid_count > current_message_count) {
if (current_message_count == 0)
return TRUE;
max_uid_del_count = uidlist->hdr.uid_count *
UIDLIST_UID_COMPRESS_PERCENTAGE / 100;
if ((uidlist->hdr.uid_count - current_message_count) >
max_uid_del_count)
return TRUE;
}
return FALSE;
}
int squat_uidlist_mark_having_expunges(struct squat_uidlist *uidlist,
bool update_disk)
{
uint8_t flag = 1;
size_t offset;
uidlist->check_expunges = TRUE;
if (update_disk) {
/* NOTE: we're writing this flag without locking */
offset = offsetof(struct squat_uidlist_header, uids_expunged);
if (pwrite_full(uidlist->fd, &flag, sizeof(flag), offset) < 0) {
squat_uidlist_set_syscall_error(uidlist,
"pwrite_full()");
return -1;
}
}
return 0;
}
struct squat_uidlist_compress_ctx *
squat_uidlist_compress_begin(struct squat_uidlist *uidlist,
const ARRAY_TYPE(seq_range) *existing_uids)
{
struct squat_uidlist_compress_ctx *ctx;
int fd;
ctx = i_new(struct squat_uidlist_compress_ctx, 1);
ctx->uidlist = uidlist;
ctx->tmp_path = i_strconcat(uidlist->filepath, ".tmp", NULL);
if (existing_uids != NULL) {
ctx->node_pool = pool_alloconly_create("compress node pool",
1024);
ctx->existing_uids = existing_uids;
i_array_init(&ctx->seen_uids,
I_MIN(128, array_count(existing_uids)));
}
fd = open(ctx->tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
if (fd == -1) {
ctx->failed = TRUE;
i_error("open(%s) failed: %m", ctx->tmp_path);
} else {
ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE);
o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
}
if (squat_uidlist_refresh(uidlist) < 0)
ctx->failed = TRUE;
return ctx;
}
static bool
squat_uidlist_is_expunged(struct squat_uidlist_compress_ctx *ctx, uint32_t uid)
{
if (ctx->existing_uids == NULL)
return FALSE;
return !seq_range_exists(ctx->existing_uids, uid);
}
static void
squat_uidlist_compress_add_uid(struct squat_uidlist_compress_ctx *ctx,
uint32_t uid)
{
struct uid_node *node;
if (squat_uidlist_is_expunged(ctx, uid)) {
ctx->seen_expunged = TRUE;
return;
}
if (!seq_range_exists(&ctx->seen_uids, uid)) {
if (uid > ctx->hdr.uid_max)
ctx->hdr.uid_max = uid;
ctx->hdr.uid_count++;
seq_range_array_add(&ctx->seen_uids, 0, uid);
}
node = p_new(ctx->node_pool, struct uid_node, 1);
node->prev = ctx->last_node;
node->uid = uid;
ctx->last_node = node;
}
static int
squat_uidlist_remove_expunged(struct squat_uidlist_compress_ctx *ctx,
const uint8_t *data, size_t size,
bool *all_expunged_r)
{
const uint8_t *end;
uint32_t num, prev_uid, next_uid, written_uid;
end = data + size;
p_clear(ctx->node_pool);
ctx->seen_expunged = FALSE;
ctx->last_node = NULL;
prev_uid = _squat_trie_unpack_num(&data, end);
squat_uidlist_compress_add_uid(ctx, prev_uid);
while (data != end) {
num = _squat_trie_unpack_num(&data, end);
next_uid = prev_uid + (num >> 1) + 1;
if ((num & 1) != 0) {
for (prev_uid++; prev_uid <= next_uid; prev_uid++)
squat_uidlist_compress_add_uid(ctx, prev_uid);
if (data == end)
break;
num = _squat_trie_unpack_num(&data, end);
next_uid += num + 1;
}
squat_uidlist_compress_add_uid(ctx, next_uid);
prev_uid = next_uid;
}
if (!ctx->seen_expunged) {
/* no changes */
return 0;
}
if (ctx->last_node == NULL) {
/* everything expunged */
*all_expunged_r = TRUE;
return 1;
}
/* recreate the list and write it */
buffer_set_used_size(ctx->uidlist->list_buf, 0);
if (squat_uidlist_write_range(ctx->uidlist, ctx->last_node,
&prev_uid, &written_uid, 0) < 0)
return -1;
if (squat_uidlist_write_listbuf(ctx->uidlist, ctx->output) < 0)
return -1;
*all_expunged_r = FALSE;
return 1;
}
int squat_uidlist_compress_next(struct squat_uidlist_compress_ctx *ctx,
uint32_t *uid_list_idx)
{
struct squat_uidlist *uidlist = ctx->uidlist;
const uint8_t *data, *data_start;
uint32_t size, old_offset;
int ret;
if ((*uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
uint32_t uid = *uid_list_idx & ~UID_LIST_IDX_FLAG_SINGLE;
if (ctx->uidlist->check_expunges) {
if (squat_uidlist_is_expunged(ctx, uid))
return 0;
}
return 1;
}
if (ctx->output == NULL)
return -1;
if (squat_uidlist_map_list(uidlist, *uid_list_idx, &data, &size) < 0) {
squat_trie_set_corrupted(uidlist->trie,
"corrupted uidlist index (compressing)");
return -1;
}
old_offset = *uid_list_idx;
*uid_list_idx = ctx->output->offset;
if (!ctx->uidlist->check_expunges)
ret = 0;
else {
bool all_expunged;
ret = squat_uidlist_remove_expunged(ctx, data, size,
&all_expunged);
if (ret < 0) {
ctx->failed = TRUE;
return -1;
}
if (ret > 0 && all_expunged)
return 0;
}
if (ret == 0) {
data_start = data = uidlist->const_mmap_base + old_offset;
(void)_squat_trie_unpack_num(&data, NULL);
if (o_stream_send(ctx->output, data_start,
data - data_start + size) < 0) {
ctx->failed = TRUE;
return -1;
}
}
ctx->hdr.node_count++;
return 1;
}
void squat_uidlist_compress_rollback(struct squat_uidlist_compress_ctx **_ctx)
{
struct squat_uidlist_compress_ctx *ctx = *_ctx;
*_ctx = NULL;
if (ctx->node_pool != NULL)
pool_unref(ctx->node_pool);
if (array_is_created(&ctx->seen_uids))
array_free(&ctx->seen_uids);
if (ctx->output != NULL) {
if (ctx->failed)
(void)unlink(ctx->tmp_path);
o_stream_destroy(&ctx->output);
}
i_free(ctx->tmp_path);
i_free(ctx);
}
int squat_uidlist_compress_commit(struct squat_uidlist_compress_ctx **_ctx)
{
struct squat_uidlist_compress_ctx *ctx = *_ctx;
int ret = 0;
if (ctx->failed) {
squat_uidlist_compress_rollback(_ctx);
return -1;
}
/* write the header */
ctx->hdr.uidvalidity = ctx->uidlist->uidvalidity;
ctx->hdr.used_file_size = ctx->output->offset;
if (ctx->existing_uids == NULL) {
ctx->hdr.uid_max = ctx->uidlist->hdr.uid_max;
ctx->hdr.uid_count = ctx->uidlist->hdr.uid_count;
}
o_stream_seek(ctx->output, 0);
if (o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr)) < 0)
ret = -1;
if (ret == 0) {
if (rename(ctx->tmp_path, ctx->uidlist->filepath) < 0) {
i_error("rename(%s, %s) failed: %m",
ctx->tmp_path, ctx->uidlist->filepath);
ret = -1;
} else {
/* reopen */
ctx->uidlist->check_expunges = FALSE;
squat_uidlist_close(ctx->uidlist);
(void)squat_uidlist_open(ctx->uidlist);
}
}
if (ret < 0)
ctx->failed = TRUE;
squat_uidlist_compress_rollback(_ctx);
return ret;
}
static void
squat_uidlist_get_add_uid(struct squat_uidlist_get_context *ctx, uint32_t uid)
{
if (ctx->filter_uid_pos == 0) {
seq_range_array_add(ctx->result, 0, uid);
return;
}
if (ctx->filter_uid_pos < uid) {
seq_range_array_remove_range(ctx->result,
ctx->filter_uid_pos, uid-1);
}
ctx->filter_uid_pos = uid+1;
}
static int
squat_uidlist_get_range_list(struct squat_uidlist_get_context *ctx,
size_t offset)
{
const uint8_t *data, *end;
uint32_t size, num, prev_uid, next_uid;
if (squat_uidlist_map_list(ctx->uidlist, offset, &data, &size) < 0)
return -1;
end = data + size;
prev_uid = _squat_trie_unpack_num(&data, end);
squat_uidlist_get_add_uid(ctx, prev_uid);
while (data != end) {
num = _squat_trie_unpack_num(&data, end);
next_uid = prev_uid + (num >> 1) + 1;
if ((num & 1) != 0) {
for (prev_uid++; prev_uid <= next_uid; prev_uid++)
squat_uidlist_get_add_uid(ctx, prev_uid);
if (data == end)
break;
num = _squat_trie_unpack_num(&data, end);
next_uid += num + 1;
}
squat_uidlist_get_add_uid(ctx, next_uid);
prev_uid = next_uid;
}
return 0;
}
static int
squat_uidlist_get_ctx(struct squat_uidlist_get_context *ctx,
uint32_t uid_list_idx)
{
if ((uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
uint32_t uid = uid_list_idx & ~UID_LIST_IDX_FLAG_SINGLE;
squat_uidlist_get_add_uid(ctx, uid);
return 0;
}
return squat_uidlist_get_range_list(ctx, uid_list_idx);
}
int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
ARRAY_TYPE(seq_range) *result)
{
struct squat_uidlist_get_context ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.uidlist = uidlist;
ctx.result = result;
return squat_uidlist_get_ctx(&ctx, uid_list_idx);
}
int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
ARRAY_TYPE(seq_range) *result)
{
struct squat_uidlist_get_context ctx;
const struct seq_range *range;
unsigned int count;
memset(&ctx, 0, sizeof(ctx));
ctx.uidlist = uidlist;
ctx.result = result;
ctx.filter_uid_pos = 1;
if (squat_uidlist_get_ctx(&ctx, uid_list_idx) < 0)
return -1;
range = array_get(ctx.result, &count);
if (count > 0) {
seq_range_array_remove_range(result, ctx.filter_uid_pos,
range[count-1].seq2);
}
return 0;
}
bool squat_uidlist_want_flush(struct squat_uidlist *uidlist)
{
return uidlist->node_pool_used >= SQUAT_UIDLIST_FLUSH_THRESHOLD;
}
size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist,
unsigned int *count_r)
{
*count_r = uidlist->hdr.node_count;
return uidlist->hdr.used_file_size;
}