/* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "httpd.h"
#include "http_log.h"
#include "http_request.h"
#include "http_protocol.h"
#include "http_config.h"
#include "mod_status.h"
#include "apr.h"
#include "apr_strings.h"
#include "apr_time.h"
#include "apr_shm.h"
#define APR_WANT_STRFUNC
#include "apr_want.h"
#include "apr_general.h"
#include "ap_socache.h"
/* XXX Unfortunately, there are still many unsigned ints in use here, so we
* XXX cannot allow more than UINT_MAX. Since some of the ints are exposed in
* XXX public interfaces, a simple search and replace is not enough.
* XXX It should be possible to extend that so that the total cache size can
* XXX be APR_SIZE_MAX and only the object size needs to be smaller than
* XXX UINT_MAX.
*/
/*
* Header structure - the start of the shared-mem segment
*/
typedef struct {
/* Stats for cache operations */
unsigned long stat_stores;
unsigned long stat_replaced;
unsigned long stat_expiries;
unsigned long stat_scrolled;
unsigned long stat_retrieves_hit;
unsigned long stat_retrieves_miss;
unsigned long stat_removes_hit;
unsigned long stat_removes_miss;
/* Number of subcaches */
unsigned int subcache_num;
/* How many indexes each subcache's queue has */
unsigned int index_num;
/* How large each subcache is, including the queue and data */
unsigned int subcache_size;
/* How far into each subcache the data area is (optimisation) */
unsigned int subcache_data_offset;
/* How large the data area in each subcache is (optimisation) */
unsigned int subcache_data_size;
} SHMCBHeader;
/*
* Subcache structure - the start of each subcache, followed by
* indexes then data
*/
typedef struct {
/* The start position and length of the cyclic buffer of indexes */
/* Same for the data area */
/*
* Index structure - each subcache has an array of these
*/
typedef struct {
/* absolute time this entry expires */
/* location within the subcache's data area */
unsigned int data_pos;
/* size (most logic ignores this, we keep it only to minimise memcpy) */
unsigned int data_used;
/* length of the used data which contains the id */
unsigned int id_len;
/* Used to mark explicitly-removed socache entries */
unsigned char removed;
} SHMCBIndex;
struct ap_socache_instance_t {
const char *data_file;
};
/* The SHM data segment is of fixed size and stores data as follows.
*
* [ SHMCBHeader | Subcaches ]
*
* The SHMCBHeader header structure stores metadata concerning the
* cache and the contained subcaches.
*
* Subcaches is a hash table of header->subcache_num SHMCBSubcache
* structures. The hash table is indexed by SHMCB_MASK(id). Each
* SHMCBSubcache structure has a fixed size (header->subcache_size),
* which is determined at creation time, and looks like the following:
*
* [ SHMCBSubcache | Indexes | Data ]
*
* Each subcache is prefixed by the SHMCBSubcache structure.
*
* The subcache's "Data" segment is a single cyclic data buffer, of
* total size header->subcache_data_size; data inside is referenced
* using byte offsets. The offset marking the beginning of the cyclic
* buffer is subcache->data_pos; the buffer's length is
* subcache->data_used.
*
* "Indexes" is an array of header->index_num SHMCBIndex structures,
* which is used as a cyclic queue; subcache->idx_pos gives the array
* index of the first in use, subcache->idx_used gives the number in
* use. Both ->idx_* values have a range of [0, header->index_num)
*
* Each in-use SHMCBIndex structure represents a single cached object.
* The ID and data segment are stored consecutively in the subcache's
* cyclic data buffer. The "Data" segment can thus be seen to
* look like this, for example
*
* offset: [ 0 1 2 3 4 5 6 ...
* contents:[ ID1 Data1 ID2 Data2 ID3 ...
*
* where the corresponding indices would look like:
*
* idx1 = { data_pos = 0, data_used = 3, id_len = 1, ...}
* idx2 = { data_pos = 3, data_used = 3, id_len = 1, ...}
* ...
*/
/* This macro takes a pointer to the header and a zero-based index and returns
* a pointer to the corresponding subcache. */
(SHMCBSubcache *)(((unsigned char *)(pHeader)) + \
/* This macro takes a pointer to the header and an id and returns a
* pointer to the corresponding subcache. */
/* This macro takes the same params as the last, generating two outputs for use
* in ap_log_error(...). */
/* This macro takes a pointer to a subcache and a zero-based index and returns
* a pointer to the corresponding SHMCBIndex. */
(SHMCBIndex *)(((unsigned char *)pSubcache) + \
(num) * ALIGNED_INDEX_SIZE)
/* This macro takes a pointer to the header and a subcache and returns a
* pointer to the corresponding data area. */
/*
* Cyclic functions - assists in "wrap-around"/modulo logic
*/
/* Addition modulo 'mod' */
/* Subtraction (or "distance between") modulo 'mod' */
/* A "normal-to-cyclic" memcpy. */
unsigned int dest_offset, const unsigned char *src,
unsigned int src_len)
{
/* It be copied all in one go */
else {
/* Copy the two splits */
}
}
/* A "cyclic-to-normal" memcpy. */
const unsigned char *data, unsigned int src_offset,
unsigned int src_len)
{
/* It be copied all in one go */
else {
/* Copy the two splits */
}
}
/* A memcmp against a cyclic data buffer. Compares SRC of length
* SRC_LEN against the contents of cyclic buffer DATA (which is of
* size BUF_SIZE), starting at offset DEST_OFFSET. Got that? Good. */
unsigned int dest_offset,
const unsigned char *src,
unsigned int src_len)
{
/* It be compared all in one go */
else {
/* Compare the two splits */
int diff;
if (diff) {
return diff;
}
}
}
/* Prototypes for low-level subcache operations */
/* Returns zero on success, non-zero on failure. */
/* Returns zero on success, non-zero on failure. */
/* Returns zero on success, non-zero on failure. */
const unsigned char *, unsigned int);
/* Returns result of the (iterator)() call, zero is success (continue) */
server_rec *s,
void *userctx,
unsigned char **buf,
/*
* High-Level "handlers" as per ssl_scache.c
* subcache internals are deferred to shmcb_subcache_*** functions lower down
*/
const char *arg,
{
/* Allocate the context. */
/* Use defaults. */
return NULL;
}
if (cp) {
char *endptr;
if (*cp2 != ')') {
return "Invalid argument: no closing parenthesis or cache size "
"missing after pathname with parenthesis";
}
*cp++ = '\0';
*cp2 = '\0';
return "Invalid argument: cache size not numerical";
}
return "Invalid argument: size has to be >= 8192 bytes";
}
}
}
return "Invalid argument: no opening parenthesis";
}
return NULL;
}
const char *namespace,
const struct ap_socache_hints *hints,
server_rec *s, apr_pool_t *p)
{
void *shm_segment;
/* Create shared memory segment */
}
/* Use anonymous shm by default, fall back on name-based. */
if (APR_STATUS_IS_ENOTIMPL(rv)) {
/* If anon shm isn't supported, fail if no named file was
* configured successfully; the ap_runtime_dir_relative call
* above will return NULL for invalid paths. */
"Could not use default path '%s' for shmcb socache",
return APR_EINVAL;
}
/* For a name-based segment, remove it first in case of a
* previous unclean shutdown. */
}
if (rv != APR_SUCCESS) {
"Could not allocate shared memory segment for shmcb "
"socache");
return rv;
}
/* the segment is ridiculously small, bail out */
"shared memory segment too small");
return APR_ENOSPC;
}
"shmcb_init allocated %" APR_SIZE_T_FMT
" bytes of shared memory",
/* Discount the header */
/* Select index size based on average object size hints, if given. */
num_subcache = 256;
num_subcache /= 2;
num_idx /= num_subcache;
" including header), recommending %u subcaches, "
"%u indexes each", shm_segsize,
if (num_idx < 5) {
/* we're still too small, bail out */
"shared memory segment too small");
return APR_ENOSPC;
}
/* OK, we're sorted */
header->stat_stores = 0;
header->stat_replaced = 0;
header->stat_expiries = 0;
header->stat_scrolled = 0;
header->stat_retrieves_hit = 0;
header->stat_retrieves_miss = 0;
header->stat_removes_hit = 0;
header->stat_removes_miss = 0;
/* Convert the subcache size (in bytes) to a value that is suitable for
* structure alignment on the host platform, by rounding down if necessary. */
}
/* Output trace info */
"shmcb_init_memory choices follow");
/* The header is done, make the caches empty */
}
"Shared memory socache initialised");
/* Success ... */
return APR_SUCCESS;
}
{
}
}
server_rec *s, const unsigned char *id,
unsigned char *encoded,
unsigned int len_encoded,
apr_pool_t *p)
{
int tryreplace;
"socache_shmcb_store (0x%02x -> subcache %d)",
/* XXX: Says who? Why shouldn't this be acceptable, or padded if not? */
if (idlen < 4) {
"(%u bytes)", idlen);
return APR_EINVAL;
}
"can't store an socache entry!");
return APR_ENOSPC;
}
if (tryreplace == 0) {
header->stat_replaced++;
}
else {
header->stat_stores++;
}
"leaving socache_shmcb_store successfully");
return APR_SUCCESS;
}
server_rec *s,
apr_pool_t *p)
{
int rv;
"socache_shmcb_retrieve (0x%02x -> subcache %d)",
/* Get the entry corresponding to the id, if it exists. */
if (rv == 0)
else
"leaving socache_shmcb_retrieve successfully");
}
server_rec *s, const unsigned char *id,
unsigned int idlen, apr_pool_t *p)
{
"socache_shmcb_remove (0x%02x -> subcache %d)",
if (idlen < 4) {
"(%u bytes)", idlen);
return APR_EINVAL;
}
rv = APR_SUCCESS;
} else {
rv = APR_NOTFOUND;
}
"leaving socache_shmcb_remove successfully");
return rv;
}
request_rec *r, int flags)
{
server_rec *s = r->server;
double expiry_total = 0;
/* Perform the iteration inside the mutex to avoid corruption or invalid
* pointer arithmetic. The rest of our logic uses read-only header data so
* doesn't need the lock. */
/* Iterate over the subcaches */
expiry_total += (double)idx_expiry;
if (!min_expiry)
else
}
}
/* Generate Output */
if (!(flags & AP_STATUS_SHORT)) {
"bytes, current entries: <b>%d</b><br>",
ap_rprintf(r, "subcaches: <b>%d</b>, indexes per subcache: <b>%d</b><br>",
if (non_empty_subcaches) {
ap_rprintf(r, "time left on oldest entries' objects: ");
if (now < average_expiry)
ap_rprintf(r, "avg: <b>%d</b> seconds, (range: %d...%d)<br>",
else
ap_rprintf(r, "expiry_threshold: <b>Calculation error!</b><br>");
}
ap_rprintf(r, "index usage: <b>%d%%</b>, cache usage: <b>%d%%</b><br>",
ap_rprintf(r, "total entries stored since starting: <b>%lu</b><br>",
ap_rprintf(r, "total entries replaced since starting: <b>%lu</b><br>",
ap_rprintf(r, "total entries expired since starting: <b>%lu</b><br>",
ap_rprintf(r, "total (pre-expiry) entries scrolled out of the cache: "
ap_rprintf(r, "total retrieves since starting: <b>%lu</b> hit, "
ap_rprintf(r, "total removes since starting: <b>%lu</b> hit, "
}
else {
ap_rputs("CacheType: SHMCB\n", r);
if (non_empty_subcaches) {
if (now < average_expiry) {
}
}
}
}
server_rec *s, void *userctx,
{
unsigned int loop;
/* Perform the iteration inside the mutex to avoid corruption or invalid
* pointer arithmetic. The rest of our logic uses read-only header data so
* doesn't need the lock. */
/* Iterate over the subcaches */
}
return rv;
}
/*
* Subcache-level cache operations
*/
{
freed++;
expired++;
else
/* not removed and not expired yet, we're done iterating */
break;
loop++;
}
if (!loop)
/* Nothing to do */
return;
"expiring %u and reclaiming %u removed socache entries",
/* We're expiring everything, piece of cake */
} else {
/* There remain other indexes, so we can use idx to adjust 'data' */
/* Adjust the indexes */
/* Adjust the data area */
}
}
{
/* Sanity check the input */
"inserting socache entry larger (%d) than subcache data area (%d)",
return -1;
}
/* First reclaim space from removed and expired records. */
/* Loop until there is enough space to insert
* XXX: This should first compress out-of-order expiries and
* removed records, and then force-remove oldest-first
*/
unsigned int loop = 0;
"about to force-expire, subcache: idx_used=%d, "
do {
/* Adjust the indexes by one */
/* There's nothing left */
break;
}
/* Adjust the data */
/* Stats */
header->stat_scrolled++;
/* Loop admin */
loop++;
"finished force-expire, subcache: idx_used=%d, "
}
/* HERE WE ASSUME THAT THE NEW ENTRY SHOULD GO ON THE END! I'M NOT
* CHECKING WHETHER IT SHOULD BE GENUINELY "INSERTED" SOMEWHERE.
*
* We aught to fix that. httpd (never mind third party modules)
* does not promise to perform any processing in date order
* (c.f. FAQ "My log entries are not in date order!")
*/
/* Insert the id */
/* Insert the data */
/* Insert the index */
"insert happened at idx=%d, data=(%u:%u)", new_idx,
return 0;
}
{
unsigned int pos;
unsigned int loop = 0;
/* Only consider 'idx' if the id matches, and the "removed"
* flag isn't set, and the record is not expired.
* Check the data length too to avoid a buffer overflow
* in case of corruption, which should be impossible,
* but it's cheap to be safe. */
unsigned int data_offset;
/* Find the offset of the data segment, after the id */
/* Copy out the data */
data_offset, *destlen);
return 0;
}
else {
/* Already stale, quietly remove and treat as not-found */
header->stat_expiries++;
"shmcb_subcache_retrieve discarding expired entry");
return -1;
}
}
/* Increment */
loop++;
}
"shmcb_subcache_retrieve found no match");
return -1;
}
const unsigned char *id,
unsigned int idlen)
{
unsigned int pos;
unsigned int loop = 0;
/* Only consider 'idx' if the id matches, and the "removed"
* flag isn't set. */
/* Found the matching entry, remove it quietly. */
"shmcb_subcache_remove removing matching entry");
return 0;
}
/* Increment */
loop++;
}
return -1; /* failure */
}
server_rec *s,
void *userctx,
unsigned char **buf,
{
unsigned int pos;
unsigned int loop = 0;
/* Only consider 'idx' if the "removed" flag isn't set. */
unsigned char *dest;
/* Find the offset of the data segment, after the id */
/* Grow to ~150% of this buffer requirement on resize
* always using APR_ALIGN_DEFAULT sized pages
*/
}
/* Copy out the data, because it's potentially cyclic */
"shmcb entry iterated");
if (rv != APR_SUCCESS)
return rv;
}
else {
/* Already stale, quietly remove and treat as not-found */
header->stat_expiries++;
"shmcb_subcache_iterate discarding expired entry");
}
}
/* Increment */
loop++;
}
return APR_SUCCESS;
}
"shmcb",
};
{
/* Also register shmcb under the default provider name. */
}
};