journal-file.c revision d98cc1f29fbf31ccc500d6e20c29b636b9af7e0f
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering This file is part of systemd.
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering Copyright 2011 Lennart Poettering
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering systemd is free software; you can redistribute it and/or modify it
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering under the terms of the GNU Lesser General Public License as published by
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering the Free Software Foundation; either version 2.1 of the License, or
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering (at your option) any later version.
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering systemd is distributed in the hope that it will be useful, but
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering WITHOUT ANY WARRANTY; without even the implied warranty of
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering Lesser General Public License for more details.
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering You should have received a copy of the GNU Lesser General Public License
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering along with systemd; If not, see <http://www.gnu.org/licenses/>.
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering#define DEFAULT_WINDOW_SIZE (8ULL*1024ULL*1024ULL)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering#define COMPRESSION_SIZE_THRESHOLD (512ULL)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering/* This is the minimum journal file size */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering#define JOURNAL_FILE_SIZE_MIN (64ULL*1024ULL) /* 64 KiB */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering/* These are the lower and upper bounds if we deduce the max_use value
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering * from the file system size */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering#define DEFAULT_MAX_USE_LOWER (1ULL*1024ULL*1024ULL) /* 1 MiB */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering#define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering/* This is the upper bound if we deduce max_size from max_use */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering#define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL) /* 128 MiB */
3f9da416457c4265b8f1179516a32ad1a987ff7dLennart Poettering/* This is the upper bound if we deduce the keep_free value from the
4f10118016f9b2fd7e1d26c9ef7d91eb33fba694Lennart Poettering * file system size */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering#define DEFAULT_KEEP_FREE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering/* This is the keep_free value when we can't determine the system
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering#define DEFAULT_KEEP_FREE (1024ULL*1024ULL) /* 1 MB */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering/* n_data was the first entry we added after the initial file format design */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering#define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data))
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering#define JOURNAL_HEADER_CONTAINS(h, field) \
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering (le64toh((h)->header_size) >= offsetof(Header, field) + sizeof((h)->field))
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_maybe_append_tag(JournalFile *f, uint64_t realtime);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_hmac_put_object(JournalFile *f, int type, uint64_t p);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering /* Write the final tag */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering /* Sync everything to disk, before we mark the file offline */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering for (t = 0; t < _WINDOW_MAX; t++)
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering munmap(f->windows[t].ptr, f->windows[t].size);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Mark the file offline. Don't override the archived state if it already is set */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (f->writable && f->header->state == STATE_ONLINE)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering munmap(f->header, PAGE_ALIGN(sizeof(Header)));
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering munmap(f->fsprg_header, PAGE_ALIGN(f->fsprg_size));
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_init_header(JournalFile *f, JournalFile *template) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering memcpy(h.signature, HEADER_SIGNATURE, 8);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering h.header_size = htole64(ALIGN64(sizeof(h)));
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering htole32(f->compress ? HEADER_INCOMPATIBLE_COMPRESSED : 0);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering htole32(f->authenticate ? HEADER_COMPATIBLE_AUTHENTICATED : 0);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering h.seqnum_id = template->header->seqnum_id;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering h.tail_seqnum = template->header->tail_seqnum;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (k != sizeof(h))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int journal_file_refresh_header(JournalFile *f) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = sd_id128_get_machine(&f->header->machine_id);
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack if (sd_id128_equal(boot_id, f->header->boot_id))
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack /* Sync the online state to disk */
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack msync(f->header, PAGE_ALIGN(sizeof(Header)), MS_SYNC);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int journal_file_verify_header(JournalFile *f) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (memcmp(f->header->signature, HEADER_SIGNATURE, 8))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* In both read and write mode we refuse to open files with
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering * incompatible flags we don't know */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if ((le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* When open for writing we refuse to open files with
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering * compatible flags, too */
79008bddf679a5e0900369950eb346c9fa687107Lennart Poettering if ((le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_AUTHENTICATED) != 0)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* The first addition was n_data, so check that we are at least this large */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (le64toh(f->header->header_size) < HEADER_SIZE_MIN)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if ((uint64_t) f->last_stat.st_size < (le64toh(f->header->header_size) + le64toh(f->header->arena_size)))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (!sd_id128_equal(machine_id, f->header->machine_id))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering log_debug("Journal file %s is already online. Assuming unclean closing.", f->path);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering log_debug("Journal file %s has unknown state %u.", f->path, state);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->compress = !!(le32toh(f->header->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->authenticate = !!(le32toh(f->header->compatible_flags) & HEADER_COMPATIBLE_AUTHENTICATED);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering /* We assume that this file is not sparse, and we know that
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * for sure, since we always call posix_fallocate()
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * ourselves */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (new_size < le64toh(f->header->header_size))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering new_size = le64toh(f->header->header_size);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Note that the glibc fallocate() fallback is very
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering inefficient, hence we try to minimize the allocation area
8f077bf94e129fa1b6f0159e3140c4326f1066cfZbigniew Jędrzejewski-Szmek r = posix_fallocate(f->fd, old_size, new_size - old_size);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->arena_size = htole64(new_size - le64toh(f->header->header_size));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering woffset = offset & ~((uint64_t) page_size() - 1ULL);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Avoid SIGBUS on invalid accesses */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (woffset + wsize > (uint64_t) PAGE_ALIGN(f->last_stat.st_size))
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering window = mmap(NULL, wsize, f->prot, MAP_SHARED, f->fd, woffset);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering *ret = (uint8_t*) window + (offset - woffset);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_move_to(JournalFile *f, int wt, uint64_t offset, uint64_t size, void **ret) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering if (offset + size > (uint64_t) f->last_stat.st_size) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Hmm, out of range? Let's refresh the fstat() data
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering * first, before we trust that check. */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering offset + size > (uint64_t) f->last_stat.st_size)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering *ret = (uint8_t*) w->ptr + (offset - w->offset);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering /* If the default window size is larger then what was
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * asked for extend the mapping a bit in the hope to
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * minimize needed remappings later on. We add half
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * the window space before and half behind the
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * requested mapping */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering delta = (DEFAULT_WINDOW_SIZE - size) / 2;
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering if (offset + size > (uint64_t) f->last_stat.st_size)
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering size = (uint64_t) f->last_stat.st_size - offset;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (o->object.type == OBJECT_DATA && !(o->object.flags & OBJECT_COMPRESSED)) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering h2 = hash64(o->data.payload, le64toh(o->object.size) - offsetof(Object, data.payload));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering } else if (o->object.type == OBJECT_FIELD) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering h2 = hash64(o->field.payload, le64toh(o->object.size) - offsetof(Object, field.payload));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringint journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_move_to(f, type >= 0 ? type : WINDOW_UNKNOWN, offset, sizeof(ObjectHeader), &t);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (s < sizeof(ObjectHeader))
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering if (s > sizeof(ObjectHeader)) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_move_to(f, o->object.type, offset, s, &t);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic uint64_t journal_file_entry_seqnum(JournalFile *f, uint64_t *seqnum) {
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack /* If an external seqnum counter was passed, we update
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack * both the local and the external one, and set it to
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mack * the maximum of both */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_append_object(JournalFile *f, int type, uint64_t size, Object **ret, uint64_t *offset) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering p = le64toh(f->header->tail_object_offset);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_move_to_object(f, -1, p, &tail);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering p += ALIGN64(le64toh(tail->object.size));
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_move_to(f, type, p, size, &t);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->tail_object_offset = htole64(p);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->n_objects = htole64(le64toh(f->header->n_objects) + 1);
5892a914d173e4b968d2a14fbf717373dee3999aDaniel Mackstatic int journal_file_setup_data_hash_table(JournalFile *f) {
79008bddf679a5e0900369950eb346c9fa687107Lennart Poettering /* We estimate that we need 1 hash table entry per 768 of
16ac401407959cbc62312e61c2dd76dbc3a0793bLennart Poettering journal file and we want to make sure we never get beyond
16ac401407959cbc62312e61c2dd76dbc3a0793bLennart Poettering 75% fill level. Calculate the hash table size for the
ff975efb2e88dcd5221a2f0d76c4c87e85b821a8Lennart Poettering maximum file size based on these metrics. */
16ac401407959cbc62312e61c2dd76dbc3a0793bLennart Poettering s = (f->metrics.max_size * 4 / 768 / 3) * sizeof(HashItem);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering log_info("Reserving %llu entries in hash table.", (unsigned long long) (s / sizeof(HashItem)));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->data_hash_table_offset = htole64(p + offsetof(Object, hash_table.items));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->data_hash_table_size = htole64(s);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int journal_file_setup_field_hash_table(JournalFile *f) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering f->header->field_hash_table_offset = htole64(p + offsetof(Object, hash_table.items));
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering f->header->field_hash_table_size = htole64(s);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int journal_file_map_data_hash_table(JournalFile *f) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering p = le64toh(f->header->data_hash_table_offset);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering s = le64toh(f->header->data_hash_table_size);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_map_field_hash_table(JournalFile *f) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering p = le64toh(f->header->field_hash_table_offset);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering s = le64toh(f->header->field_hash_table_size);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int journal_file_link_data(JournalFile *f, Object *o, uint64_t offset, uint64_t hash) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* This might alter the window we are looking at */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->data.next_hash_offset = o->data.next_field_offset = 0;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->data.entry_offset = o->data.entry_array_offset = 0;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering h = hash % (le64toh(f->header->data_hash_table_size) / sizeof(HashItem));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering p = le64toh(f->data_hash_table[h].tail_hash_offset);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Only entry in the hash table is easy */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->data_hash_table[h].head_hash_offset = htole64(offset);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Move back to the previous data object, to patch in
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->data.next_hash_offset = htole64(offset);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->data_hash_table[h].tail_hash_offset = htole64(offset);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (JOURNAL_HEADER_CONTAINS(f->header, n_data))
79008bddf679a5e0900369950eb346c9fa687107Lennart Poettering f->header->n_data = htole64(le64toh(f->header->n_data) + 1);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringint journal_file_find_data_object_with_hash(
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering const void *data, uint64_t size, uint64_t hash,
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering osize = offsetof(Object, data.payload) + size;
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering if (f->header->data_hash_table_size == 0)
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering h = hash % (le64toh(f->header->data_hash_table_size) / sizeof(HashItem));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering p = le64toh(f->data_hash_table[h].head_hash_offset);
79008bddf679a5e0900369950eb346c9fa687107Lennart Poettering while (p > 0) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (o->object.flags & OBJECT_COMPRESSED) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize))
bd5f920f1288c0d4d488629fadf067f709227030Lennart Poettering memcmp(f->compress_buffer, data, size) == 0) {
dcc2fc01fa850e9ee36c549dc2691e7e5c71bebfLennart Poettering } else if (le64toh(o->object.size) == osize &&
dcc2fc01fa850e9ee36c549dc2691e7e5c71bebfLennart Poettering memcmp(o->data.payload, data, size) == 0) {
bd5f920f1288c0d4d488629fadf067f709227030Lennart Poettering return journal_file_find_data_object_with_hash(f,
bd5f920f1288c0d4d488629fadf067f709227030Lennart Poettering r = journal_file_find_data_object_with_hash(f, data, size, hash, &o, &p);
bd5f920f1288c0d4d488629fadf067f709227030Lennart Poettering else if (r > 0) {
bd5f920f1288c0d4d488629fadf067f709227030Lennart Poettering osize = offsetof(Object, data.payload) + size;
bd5f920f1288c0d4d488629fadf067f709227030Lennart Poettering r = journal_file_append_object(f, OBJECT_DATA, osize, &o, &p);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering compressed = compress_blob(data, size, o->data.payload, &rsize);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->object.size = htole64(offsetof(Object, data.payload) + rsize);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering log_debug("Compressed data object %lu -> %lu", (unsigned long) size, (unsigned long) rsize);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_link_data(f, o, p, hash);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_hmac_put_object(f, OBJECT_DATA, p);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering /* The linking might have altered the window, so let's
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering * refresh our pointer */
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringuint64_t journal_file_entry_n_items(Object *o) {
79008bddf679a5e0900369950eb346c9fa687107Lennart Poettering return (le64toh(o->object.size) - offsetof(Object, entry.items)) / sizeof(EntryItem);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic uint64_t journal_file_entry_array_n_items(Object *o) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering assert(o->object.type == OBJECT_ENTRY_ARRAY);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering return (le64toh(o->object.size) - offsetof(Object, entry_array.items)) / sizeof(uint64_t);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int link_entry_into_array(JournalFile *f,
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering while (a > 0) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_move_to_object(f, OBJECT_ENTRY_ARRAY, a, &o);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering a = le64toh(o->entry_array.next_entry_array_offset);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_append_object(f, OBJECT_ENTRY_ARRAY,
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering offsetof(Object, entry_array.items) + n * sizeof(uint64_t),
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = journal_file_hmac_put_object(f, OBJECT_ENTRY_ARRAY, q);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_move_to_object(f, OBJECT_ENTRY_ARRAY, ap, &o);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->entry_array.next_entry_array_offset = htole64(q);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poetteringstatic int link_entry_into_array_plus_one(JournalFile *f,
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering r = link_entry_into_array(f, first, &i, p);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_link_entry_item(JournalFile *f, Object *o, uint64_t offset, uint64_t i) {
0faacd470dfbd24f4c6504da6f04213aa05f9d19Lennart Poettering p = le64toh(o->entry.items[i].object_offset);
0faacd470dfbd24f4c6504da6f04213aa05f9d19Lennart Poettering r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_link_entry(JournalFile *f, Object *o, uint64_t offset) {
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poettering /* Link up the entry itself */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* log_debug("=> %s seqnr=%lu n_entries=%lu", f->path, (unsigned long) o->entry.seqnum, (unsigned long) f->header->n_entries); */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->head_entry_realtime = o->entry.realtime;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->tail_entry_realtime = o->entry.realtime;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering f->header->tail_entry_monotonic = o->entry.monotonic;
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering /* Link up the items */
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering for (i = 0; i < n; i++) {
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_link_entry_item(f, o, offset, i);
a4152e3fe28b53b8919cc404dd7eca7ead1bf9bdLennart Poetteringstatic int journal_file_append_entry_internal(
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering const EntryItem items[], unsigned n_items,
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering osize = offsetof(Object, entry.items) + (n_items * sizeof(EntryItem));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_append_object(f, OBJECT_ENTRY, osize, &o, &np);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->entry.seqnum = htole64(journal_file_entry_seqnum(f, seqnum));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering memcpy(o->entry.items, items, n_items * sizeof(EntryItem));
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->entry.realtime = htole64(ts->realtime);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering o->entry.monotonic = htole64(ts->monotonic);
e821075a23fdfa3ca7738fc30bb2d4c430fe10c0Lennart Poettering r = journal_file_hmac_put_object(f, OBJECT_ENTRY, np);
if (ret)
*ret = o;
if (offset)
assert(f);
int journal_file_append_entry(JournalFile *f, const dual_timestamp *ts, const struct iovec iovec[], unsigned n_iovec, uint64_t *seqnum, Object **ret, uint64_t *offset) {
assert(f);
if (!f->writable)
return -EPERM;
if (!ts) {
if (f->tail_entry_monotonic_valid &&
return -EINVAL;
for (i = 0; i < n_iovec; i++) {
uint64_t p;
Object *o;
uint64_t i,
Object *o;
uint64_t p = 0, a;
assert(f);
a = first;
uint64_t n;
n = journal_file_entry_array_n_items(o);
if (ret)
*ret = o;
if (offset)
*offset = p;
uint64_t i,
Object *o;
assert(f);
if (ret)
*ret = o;
if (offset)
uint64_t n,
bool subtract_one = false;
assert(f);
a = first;
if (right <= 0)
return -EBADMSG;
if (r == TEST_FOUND)
if (r == TEST_RIGHT) {
left = 0;
subtract_one = true;
i = left;
goto found;
return -EBADMSG;
if (r == TEST_FOUND)
if (r == TEST_RIGHT)
right = i;
subtract_one = true;
goto found;
if (subtract_one && t == 0 && i == 0)
if (subtract_one && i == 0)
p = last_p;
else if (subtract_one)
if (ret)
*ret = o;
if (offset)
*offset = p;
if (idx)
uint64_t n,
bool step_back = false;
Object *o;
assert(f);
if (r == TEST_FOUND)
if (r == TEST_LEFT)
if (r == TEST_RIGHT) {
goto found;
if (r == 0 && step_back)
goto found;
if (r > 0 && idx)
(*idx) ++;
if (ret)
*ret = o;
if (offset)
if (idx)
*idx = 0;
assert(f);
assert(p > 0);
if (p == needle)
return TEST_FOUND;
else if (p < needle)
return TEST_LEFT;
return TEST_RIGHT;
JournalFile *f,
uint64_t p,
return generic_array_bisect(f,
Object *o;
assert(f);
assert(p > 0);
return TEST_FOUND;
return TEST_LEFT;
return TEST_RIGHT;
JournalFile *f,
return generic_array_bisect(f,
Object *o;
assert(f);
assert(p > 0);
return TEST_FOUND;
return TEST_LEFT;
return TEST_RIGHT;
JournalFile *f,
return generic_array_bisect(f,
Object *o;
assert(f);
assert(p > 0);
return TEST_FOUND;
return TEST_LEFT;
return TEST_RIGHT;
JournalFile *f,
Object *o;
assert(f);
return -ENOENT;
return generic_array_bisect_plus_one(f,
JournalFile *f,
uint64_t i, n;
assert(f);
assert(p > 0 || !o);
return -EINVAL;
r = generic_array_bisect(f,
return generic_array_get(f,
JournalFile *f,
uint64_t i, n;
assert(f);
assert(o);
assert(p > 0);
return -EINVAL;
r = generic_array_bisect(f,
if (skip < 0) {
return -EBADMSG;
return generic_array_get(f,
JournalFile *f,
uint64_t n, i;
Object *d;
assert(f);
assert(p > 0 || !o);
return -EINVAL;
return generic_array_get_plus_one(f,
JournalFile *f,
uint64_t p,
Object *d;
assert(f);
return generic_array_bisect_plus_one(f,
JournalFile *f,
Object *o, *d;
uint64_t b, z;
assert(f);
return -ENOENT;
uint64_t p, q;
if (ret)
if (offset)
*offset = q;
JournalFile *f,
Object *d;
assert(f);
return generic_array_bisect_plus_one(f,
JournalFile *f,
Object *d;
assert(f);
return generic_array_bisect_plus_one(f,
uint64_t a, b;
assert(f);
if (!f->authenticate)
return NULL;
if (a + b > f->fsprg_size)
return NULL;
uint64_t r;
assert(f);
Object *o;
uint64_t p;
assert(f);
if (!f->authenticate)
if (!f->hmac_running)
f->hmac_running = false;
assert(f);
if (!f->authenticate)
if (f->hmac_running)
f->hmac_running = true;
uint64_t t;
assert(f);
return -ENOTSUP;
return -ESTALE;
*epoch = t;
assert(f);
if (!f->authenticate)
return -ESTALE;
assert(f);
if (!f->authenticate)
log_debug("Evolving FSPRG key from epoch %llu to %llu.", (unsigned long long) epoch, (unsigned long long) goal);
return -ESTALE;
assert(f);
if (!f->authenticate)
r = journal_file_append_tag(f);
r = journal_file_hmac_start(f);
Object *o;
assert(f);
if (!f->authenticate)
r = journal_file_hmac_start(f);
case OBJECT_DATA:
case OBJECT_ENTRY:
case OBJECT_FIELD_HASH_TABLE:
case OBJECT_DATA_HASH_TABLE:
case OBJECT_ENTRY_ARRAY:
case OBJECT_TAG:
return -EINVAL;
assert(f);
if (!f->authenticate)
r = journal_file_hmac_start(f);
gcry_md_write(f->hmac, f->header->signature, offsetof(Header, state) - offsetof(Header, signature));
gcry_md_write(f->hmac, &f->header->seqnum_id, offsetof(Header, arena_size) - offsetof(Header, seqnum_id));
gcry_md_write(f->hmac, &f->header->data_hash_table_offset, offsetof(Header, tail_object_offset) - offsetof(Header, data_hash_table_offset));
gcry_md_write(f->hmac, &f->header->head_seqnum, offsetof(Header, head_entry_realtime) - offsetof(Header, head_seqnum));
char *p = NULL;
assert(f);
if (!f->authenticate)
return -ENOMEM;
if (fd < 0) {
r = -errno;
goto finish;
r = -errno;
goto finish;
r = -ENODATA;
goto finish;
if (m == MAP_FAILED) {
m = NULL;
r = -errno;
goto finish;
r = -EBADMSG;
goto finish;
if (m->incompatible_flags != 0) {
r = -EPROTONOSUPPORT;
goto finish;
r = -EBADMSG;
goto finish;
r = -EBADMSG;
goto finish;
r = -ENODATA;
goto finish;
r = -EHOSTDOWN;
goto finish;
r = -EBADMSG;
goto finish;
r = -errno;
goto finish;
if (fd >= 0)
free(p);
gcry_error_t e;
if (!f->authenticate)
return -ENOTSUP;
uint64_t p;
if (!f->authenticate)
r = journal_file_hmac_put_header(f);
return -EINVAL;
return -EINVAL;
r = journal_file_append_tag(f);
Object *o;
uint64_t p;
assert(f);
goto fail;
case OBJECT_UNUSED:
case OBJECT_DATA:
case OBJECT_ENTRY:
case OBJECT_FIELD_HASH_TABLE:
case OBJECT_DATA_HASH_TABLE:
case OBJECT_ENTRY_ARRAY:
case OBJECT_TAG:
fail:
assert(f);
f->path,
100.0 * (double) le64toh(f->header->n_data) / ((double) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem))));
100.0 * (double) le64toh(f->header->n_fields) / ((double) (le64toh(f->header->field_hash_table_size) / sizeof(HashItem))));
int journal_file_open(
const char *fname,
int flags,
bool compress,
bool authenticate,
JournalFile *f;
bool newly_created = false;
return -EINVAL;
return -EINVAL;
return -ENOMEM;
if (!f->path) {
r = -ENOMEM;
goto fail;
if (f->fd < 0) {
r = -errno;
goto fail;
r = -errno;
goto fail;
newly_created = true;
r = journal_file_load_fsprg(f);
f->authenticate = false;
goto fail;
r = -errno;
goto fail;
r = -EIO;
goto fail;
r = -errno;
goto fail;
if (!newly_created) {
r = journal_file_verify_header(f);
goto fail;
r = journal_file_load_fsprg(f);
goto fail;
if (f->writable) {
if (metrics) {
} else if (template)
r = journal_file_refresh_header(f);
goto fail;
r = journal_file_setup_hmac(f);
goto fail;
if (newly_created) {
goto fail;
goto fail;
r = journal_file_append_first_tag(f);
goto fail;
goto fail;
r = journal_file_map_data_hash_table(f);
goto fail;
if (ret)
*ret = f;
fail:
size_t l;
assert(f);
assert(*f);
old_file = *f;
return -EINVAL;
return -EINVAL;
return -ENOMEM;
free(p);
return -errno;
r = journal_file_open(old_file->path, old_file->flags, old_file->mode, compress, authenticate, NULL, old_file, &new_file);
*f = new_file;
const char *fname,
int flags,
bool compress,
bool authenticate,
size_t l;
random_ull()) < 0)
return -ENOMEM;
free(p);
return -errno;
struct vacuum_info {
char *filename;
bool have_seqnum;
const struct vacuum_info *a, *b;
a = _a;
b = _b;
DIR *d;
if (max_use <= 0)
return -errno;
size_t q;
bool have_seqnum;
goto finish;
if (!de)
r = -ENOMEM;
goto finish;
free(p);
free(p);
have_seqnum = true;
unsigned long long tmp;
r = -ENOMEM;
goto finish;
free(p);
have_seqnum = false;
struct vacuum_info *j;
free(p);
r = -ENOMEM;
goto finish;
list = j;
n_list ++;
if (n_list > 0)
for(i = 0; i < n_list; i++) {
r = -errno;
goto finish;
for (i = 0; i < n_list; i++)
closedir(d);
int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {
uint64_t i, n;
assert(o);
assert(p);
return -EPERM;
return -EINVAL;
n = journal_file_entry_n_items(o);
uint64_t l, h;
size_t t;
void *data;
Object *u;
return -EBADMSG;
t = (size_t) l;
if ((uint64_t) t != l)
return -E2BIG;
#ifdef HAVE_XZ
if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize))
return -EBADMSG;
l = rsize;
return -EPROTONOSUPPORT;
assert(m);
if (fs_size > 0) {
if (fs_size > 0) {
assert(f);
if (from) {
return -ENOENT;
if (to) {
return -ENOENT;
int journal_file_get_cutoff_monotonic_usec(JournalFile *f, sd_id128_t boot_id, usec_t *from, usec_t *to) {
Object *o;
uint64_t p;
assert(f);
if (from) {
if (to) {
r = generic_array_get_plus_one(f,
&o, NULL);
assert(f);
if (le64toh(f->header->n_data) * 4ULL > (le64toh(f->header->data_hash_table_size) / sizeof(HashItem)) * 3ULL) {
log_debug("Data hash table of %s has a fill level at %.1f (%llu of %llu items, %llu file size, %llu bytes per hash table item), suggesting rotation.",
f->path,
100.0 * (double) le64toh(f->header->n_data) / ((double) (le64toh(f->header->data_hash_table_size) / sizeof(HashItem))),
if (le64toh(f->header->n_fields) * 4ULL > (le64toh(f->header->field_hash_table_size) / sizeof(HashItem)) * 3ULL) {
log_debug("Field hash table of %s has a fill level at %.1f (%llu of %llu items), suggesting rotation.",
f->path,