/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
/*
* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
*/
/*
* This file contains the core framework routines for the
* kernel cryptographic framework. These routines are at the
*/
#include <sys/sysmacros.h>
#include <sys/taskq_impl.h>
/* Thread pool related variables */
/* kmem caches used by the scheduler */
/* Global request ID table */
/* KCF stats. Not protected. */
{ "total threads in pool", KSTAT_DATA_UINT32},
{ "idle threads in pool", KSTAT_DATA_UINT32},
{ "min threads in pool", KSTAT_DATA_UINT32},
{ "max threads in pool", KSTAT_DATA_UINT32},
{ "requests in gswq", KSTAT_DATA_UINT32},
{ "max requests in gswq", KSTAT_DATA_UINT32},
{ "threads for HW taskq", KSTAT_DATA_UINT32},
{ "minalloc for HW taskq", KSTAT_DATA_UINT32},
{ "maxalloc for HW taskq", KSTAT_DATA_UINT32}
};
static int kcf_disp_sw_request(kcf_areq_node_t *);
static void process_req_hwp(void *);
static kcf_areq_node_t *kcf_dequeue(void);
static int kcf_enqueue(kcf_areq_node_t *);
static void kcfpool_alloc(void);
static void compute_min_max_threads(void);
static void kcfpool_svc(void *);
static void kcfpoold(void *);
/*
* Create a new context.
*/
{
return (NULL);
/* initialize the context for the consumer */
return (ctx);
}
/*
* Allocate a new async request node.
*
* ictx - Framework private context pointer
* crq - Has callback function and argument. Should be non NULL.
* req - The parameters to pass to the SPI
*/
static kcf_areq_node_t *
{
return (NULL);
/*
* Requests for context-less operations do not use the
* fields - an_is_my_turn, and an_ctxchain_next.
*/
return (arptr);
/*
* Chain this request to the context.
*/
} else {
/* Insert the new request to the end of the chain. */
}
return (arptr);
}
/*
* Queue the request node and do one of the following:
* - If there is an idle thread signal it to run.
* - Else, signal the creator thread to possibly create more threads.
*/
static int
{
int err;
return (err);
if (kcfpool->kp_idlethreads > 0) {
/* Signal an idle thread to run */
return (CRYPTO_QUEUED);
}
/* Signal the creator thread for more threads */
return (CRYPTO_QUEUED);
}
/*
* This routine is called by the taskq associated with
* each hardware provider. We notify the kernel consumer
* via the callback routine in case of CRYPTO_SUCCESS or
* a failure.
*
* A request can be of type kcf_areq_node_t or of type
* kcf_sreq_node_t.
*/
static void
{
int error = 0;
/*
* Wait if flow control is in effect for the provider. A
* CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED
* notification will signal us. We also get signaled if
* the provider is unregistering.
*/
}
/*
* Bump the internal reference count while the request is being
* processed. This is how we know when it's safe to unregister
* a provider. This step must precede the pd_state check below.
*/
/*
* Fail the request if the provider has failed. We return a
* recoverable error and the notified clients attempt any
* recovery. For async clients this is done in kcf_aop_done()
* and for sync clients it is done in the k-api routines.
*/
goto bail;
}
if (ctype == CRYPTO_SYNCH) {
} else {
/*
* We are in the per-hardware provider thread context and
* hence can sleep. Note that the caller would have done
* a taskq_dispatch(..., TQ_NOSLEEP) and would have returned.
*/
/*
* We need to maintain ordering for multi-part requests.
* an_is_my_turn is set to B_TRUE initially for a request
* when it is enqueued and there are no other requests
* for that context. It is set later from kcf_aop_done() when
* the request before us in the chain of requests for the
* context completes. We get signaled at that point.
*/
}
}
}
bail:
if (error == CRYPTO_QUEUED) {
/*
* The request is queued by the provider and we should
* get a crypto_op_notification() from the provider later.
* We notify the consumer at that time.
*/
return;
} else { /* CRYPTO_SUCCESS or other failure */
if (ctype == CRYPTO_SYNCH)
else
}
}
/*
* This routine checks if a request can be retried on another
* provider. If true, mech1 is initialized to point to the mechanism
* structure. mech2 is also initialized in case of a dual operation. fg
* is initialized to the correct crypto_func_group_t bit flag. They are
* initialized by this routine, so that the caller can pass them to a
* kcf_get_mech_provider() or kcf_get_dual_provider() with no further change.
*
* We check that the request is for a init or atomic routine and that
* it is for one of the operation groups used from k-api .
*/
static boolean_t
{
return (B_FALSE);
case KCF_OG_DIGEST: {
break;
}
case KCF_OG_MAC: {
break;
}
case KCF_OG_SIGN: {
switch (optype) {
case KCF_OP_INIT:
*fg = CRYPTO_FG_SIGN;
break;
case KCF_OP_ATOMIC:
break;
default:
}
break;
}
case KCF_OG_VERIFY: {
switch (optype) {
case KCF_OP_INIT:
*fg = CRYPTO_FG_VERIFY;
break;
case KCF_OP_ATOMIC:
break;
default:
}
break;
}
case KCF_OG_ENCRYPT: {
break;
}
case KCF_OG_DECRYPT: {
break;
}
case KCF_OG_ENCRYPT_MAC: {
break;
}
case KCF_OG_MAC_DECRYPT: {
break;
}
default:
return (B_FALSE);
}
return (B_TRUE);
}
/*
* This routine is called when a request to a provider has failed
* with a recoverable error. This routine tries to find another provider
* and dispatches the request to the new provider, if one is available.
* We reuse the request structure.
*
* A return value of NULL from kcf_get_mech_provider() indicates
* we have tried the last provider.
*/
static int
{
return (error);
/*
* Add old_pd to the list of providers already tried.
* We release the new hold on old_pd in kcf_free_triedlist().
*/
return (error);
} else {
}
return (error);
/*
* We reuse the old context by resetting provider specific
* fields in it.
*/
}
/* We reuse areq. by resetting the provider and context fields. */
switch (new_pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
break;
case CRYPTO_HW_PROVIDER: {
(taskqid_t)0) {
} else {
}
break;
}
}
return (error);
}
/*
* Routine called by both ioctl and k-api. The consumer should
* bundle the parameters into a kcf_req_params_t structure. A bunch
* of macros are available in ops_impl.h for this bundling. They are:
*
* KCF_WRAP_DIGEST_OPS_PARAMS()
* KCF_WRAP_MAC_OPS_PARAMS()
* KCF_WRAP_ENCRYPT_OPS_PARAMS()
* KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc.
*
* It is the caller's responsibility to free the ctx argument when
* appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details.
*/
int
{
int error;
/* Synchronous cases */
switch (pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
break;
case CRYPTO_HW_PROVIDER:
/*
* Special case for CRYPTO_SYNCHRONOUS providers that
* never return a CRYPTO_QUEUED error. We skip any
* request allocation and call the SPI directly.
*/
EMPTY_TASKQ(taskq)) {
break;
}
}
/*
* Note that we do not need to hold the context
* for synchronous case as the context will never
* become invalid underneath us. We do not need to hold
* the provider here either as the caller has a hold.
*/
/*
* Call the SPI directly if the taskq is empty and the
* provider is not busy, else dispatch to the taskq.
* Calling directly is fine as this is the synchronous
* case. This is unlike the asynchronous case where we
* must always dispatch to the taskq.
*/
if (EMPTY_TASKQ(taskq) &&
} else {
/*
* We can not tell from taskq_dispatch() return
* value if we exceeded maxalloc. Hence the
* check here. Since we are allowed to wait in
* the synchronous case, we wait for the taskq
* to become empty.
*/
}
}
/*
* Wait for the notification to arrive,
* if the operation is not done yet.
* Bug# 4722589 will make the wait a cv_wait_sig().
*/
break;
default:
break;
}
} else { /* Asynchronous cases */
switch (pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
/*
* This case has less overhead since there is
* no switching of context.
*/
} else {
/*
* CRYPTO_ALWAYS_QUEUE is set. We need to
* queue the request and return.
*/
else {
& CRYPTO_SKIP_REQID)) {
/*
* Set the request handle. This handle
* is used for any crypto_cancel_req(9f)
* calls from the consumer. We have to
* do this before dispatching the
* request.
*/
}
/*
* There is an error processing this
* request. Remove the handle and
* release the request structure.
*/
if (error != CRYPTO_QUEUED) {
}
}
}
break;
case CRYPTO_HW_PROVIDER:
/*
* We need to queue the request and return.
*/
cont);
goto done;
}
/*
* We can not tell from taskq_dispatch() return
* value if we exceeded maxalloc. Hence the check
* here.
*/
error = CRYPTO_BUSY;
goto done;
}
/*
* Set the request handle. This handle is used
* for any crypto_cancel_req(9f) calls from the
* consumer. We have to do this before dispatching
* the request.
*/
}
if (taskq_dispatch(taskq,
(taskqid_t)0) {
} else {
}
break;
default:
break;
}
}
done:
return (error);
}
/*
* We're done with this framework context, so free it. Note that freeing
* framework context (kcf_context) frees the global context (crypto_ctx).
*
* The provider is responsible for freeing provider private context after a
* final or single operation and resetting the cc_provider_private field
* to NULL. It should do this before it notifies the framework of the
* completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases
* like crypto_cancel_ctx(9f).
*/
void
{
/* Release the second context, if any */
if (kcf_secondctx != NULL)
if (!KCF_IS_PROV_REMOVED(pd)) {
/*
* Increment the provider's internal refcnt so it
* doesn't unregister from the framework while
* we're calling the entry point.
*/
} else {
}
}
/* kcf_ctx->kc_prov_desc has a hold on pd */
/* check if this context is shared with a software provider */
}
}
/*
* Free the request after releasing all the holds.
*/
void
{
}
/*
* Utility routine to remove a request from the chain of requests
* hanging off a context.
*/
void
{
/*
* Get context lock, search for areq in the chain and remove it.
*/
if ((ictx->kc_req_chain_first =
} else {
}
break;
}
}
}
/*
* Remove the specified node from the global software queue.
*
* The caller must hold the queue lock and request lock (an_lock).
*/
void
{
else
else
}
/*
* Remove and return the first node in the global software queue.
*
* The caller must hold the queue lock.
*/
static kcf_areq_node_t *
kcf_dequeue(void)
{
return (NULL);
} else {
else
}
return (tnode);
}
/*
* Add the request node to the end of the global software queue.
*
* The caller should not hold the queue lock. Returns 0 if the
* request is successfully queued. Returns CRYPTO_BUSY if the limit
* on the number of jobs is exceeded.
*/
static int
{
return (CRYPTO_BUSY);
}
} else {
}
/* an_lock not needed here as we hold gs_lock */
return (0);
}
/*
* Function run by a thread from kcfpool to work on global software queue.
*/
void
{
int error = 0;
for (;;) {
switch (rv) {
case 0:
case -1:
/*
* Woke up with no work to do. Check
* if this thread should exit. We keep
* at least kcf_minthreads.
*/
/*
* lwp_exit() assumes it is called
* with the proc lock held. But the
* first thing it does is drop it.
* This ensures that lwp does not
* exit before lwp_create is done
* with it.
*/
lwp_exit(); /* does not return */
}
/* Resume the wait for work. */
break;
default:
/*
* We are signaled to work on the queue.
*/
break;
}
}
continue;
}
/*
* We check if we can work on the request now.
* Solaris does not guarantee any order on how the threads
* are scheduled or how the waiters on a mutex are chosen.
* So, we need to maintain our own order.
*
* is_my_turn is set to B_TRUE initially for a request when
* it is enqueued and there are no other requests
* for that context. Note that a thread sleeping on
* an_turn_cv is not counted as an idle thread. This is
* because we define an idle thread as one that sleeps on the
* global queue waiting for new requests.
*/
}
}
}
/*
* kmem_cache_alloc constructor for sync request structure.
*/
/* ARGSUSED */
static int
{
return (0);
}
/* ARGSUSED */
static void
{
}
/*
* kmem_cache_alloc constructor for async request structure.
*/
/* ARGSUSED */
static int
{
return (0);
}
/* ARGSUSED */
static void
{
}
/*
* kmem_cache_alloc constructor for kcf_context structure.
*/
/* ARGSUSED */
static int
{
return (0);
}
/* ARGSUSED */
static void
{
}
/*
* Creates and initializes all the structures needed by the framework.
*/
void
kcf_sched_init(void)
{
int i;
/*
* Create all the kmem caches needed by the framework. We set the
* align argument to 64, to get a slab aligned to 64-byte as well as
* have the objects (cache_chunksize) to be a 64-byte multiple.
* This helps to avoid false sharing as this is the size of the
* CPU cache line.
*/
/* Initialize the global reqid table */
for (i = 0; i < REQID_TABLES; i++) {
kcf_reqid_table[i] = rt;
}
/* Allocate and initialize the thread pool */
/* Initialize the event notification list variables */
/* Initialize the crypto_bufcall list variables */
/* Create the kcf kstat */
if (kcf_misc_kstat != NULL) {
}
}
/*
*
* kcf_sched_running flag isn't protected by a lock. But, we are safe because
* the first thread ("cryptoadm refresh") calling this routine during
* boot time completes before any other thread that can call this routine.
*/
void
kcf_sched_start(void)
{
if (kcf_sched_running)
return;
/* Start the background processing thread. */
}
/*
* Signal the waiting sync client.
*/
void
{
}
/*
* Callback the async client with the operation status.
* We free the async request node and possibly the context.
* We also handle any chain of requests hanging off of
* the context.
*/
void
{
/*
* Handle recoverable errors. This has to be done first
* before doing any thing else in this routine so that
* we do not change the state of the request.
*/
/*
* We try another provider, if one is available. Else
* we continue with the failure notification to the
* client.
*/
return;
}
/*
* A request after it is removed from the request
* queue, still stays on a chain of requests hanging
* of its context structure. It needs to be removed
* from this chain at this point.
*/
}
/*
* NOTE - We do not release the context in case of update
* operations. We require the consumer to free it explicitly,
* in case it wants to abandon an update operation. This is done
* as there may be mechanisms in ECB mode that can continue
* even if an operation on a block fails.
*/
}
}
/* Deal with the internal continuation to this request first */
return;
}
/*
* If CRYPTO_NOTIFY_OPDONE flag is set, we should notify
* always. If this flag is clear, we skip the notification
* provided there are no errors. We check this flag for only
* init or update operations. It is ignored for single, final or
* atomic operations.
*/
(error == CRYPTO_SUCCESS);
if (!skip_notify) {
}
}
/*
* kcfpool thread spawner. This runs as a process that never exits.
* Its a process so that the threads it owns can be manipulated via priocntl.
*/
static void
{
int cnt;
callb_generic_cpr, "kcfpool");
/* make our process "kcfpoold" */
/*
* Go to sleep, waiting for the signaled flag. Note that as
* we always do the same thing, and its always idempotent, we
* don't even need to have a real condition to check against.
*/
for (;;) {
int rv;
switch (rv) {
case -1:
break;
default:
/* Someone may be looking for a worker thread */
break;
}
/*
* We keep the number of running threads to be at
* kcf_minthreads to reduce gs_lock contention.
*/
cnt = kcf_minthreads -
if (cnt > 0) {
/*
* The following ensures the number of threads in pool
* does not exceed kcf_maxthreads.
*/
}
for (int i = 0; i < cnt; i++) {
(void) lwp_kernel_create(curproc,
}
}
}
/*
* Allocate the thread pool and initialize all the fields.
*/
static void
kcfpool_alloc(void)
{
kcfpool->kp_blockedthreads = 0;
/*
* Create the daemon thread.
*/
NULL, 0) != 0) {
}
}
/*
* This routine introduces a locking order for gswq->gs_lock followed
* by cpu_lock.
* This means that no consumer of the k-api should hold cpu_lock when calling
* k-api routines.
*/
static void
compute_min_max_threads(void)
{
}
/*
* Insert the async request in the hash table after assigning it
* an ID. Returns the ID.
*
* The ID is used by the caller to pass as an argument to a
* cancel_req() routine later.
*/
static crypto_req_id_t
{
int indx;
return (id);
}
/*
* Delete the async request from the hash table.
*/
static void
{
int indx;
else
}
/*
* Cancel a single asynchronous request.
*
* We guarantee that no problems will result from calling
* crypto_cancel_req() for a request which is either running, or
* has already completed. We remove the request from any queues
* if it is possible. We wait for request completion if the
* request is dispatched to a provider.
*
* Calling context:
* Can be called from user context only.
*
* NOTE: We acquire the following locks in this routine (in order):
* - rt_lock (kcf_reqid_table_t)
* - gswq->gs_lock
* - areq->an_lock
* - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain())
*
* This locking order MUST be maintained in code every where else.
*/
void
{
int indx;
/*
* We found the request. It is either still waiting
* in the framework queues or running at the provider.
*/
switch (pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
/* This request can be safely canceled. */
/* Remove from gswq, global software queue. */
/* Remove areq from hash table and free it. */
return;
}
break;
case CRYPTO_HW_PROVIDER:
/*
* There is no interface to remove an entry
* once it is on the taskq. So, we do not do
* any thing for a hardware provider.
*/
break;
}
/*
* The request is running. Wait for the request completion
* to notify us.
*/
break;
}
}
}
/*
* Cancel all asynchronous requests associated with the
* passed in crypto context and free it.
*
* A client SHOULD NOT call this routine after calling a crypto_*_final
* routine. This routine is called only during intermediate operations.
* The client should not use the crypto context after this function returns
* since we destroy it.
*
* Calling context:
* Can be called from user context only.
*/
void
{
return;
/* Walk the chain and cancel each request */
/*
* We have to drop the lock here as we may have
* to wait for request completion. We hold the
* request before dropping the lock though, so that it
* won't be freed underneath us.
*/
}
}
/*
* Update kstats.
*/
static int
{
if (rw == KSTAT_WRITE)
return (EACCES);
return (0);
}
/*
* Allocate and initiatize a kcf_dual_req, used for saving the arguments of
* a dual operation or an atomic operation that has to be internally
* simulated with multiple single steps.
* crq determines the memory allocation flags.
*/
{
return (NULL);
/* Copy the whole crypto_call_req struct, as it isn't persistent */
else
kcr->kr_saveoffset = 0;
kcr->kr_savelen = 0;
return (kcr);
}
/*
* Callback routine for the next part of a simulated dual part.
* Schedules the next step.
*
* This routine can be called from interrupt context.
*/
void
{
/* Stop the processing if an error occurred at this step */
if (error != CRYPTO_SUCCESS) {
out:
return;
}
case KCF_OG_MAC: {
/*
* The next req is submitted with the same reqid as the
* first part. The consumer only got back that reqid, and
* should still be able to cancel the operation during its
* second step.
*/
/* No expected recoverable failures, so no retry list */
goto out;
}
/* Validate the MAC context template here */
goto out;
}
}
break;
}
case KCF_OG_DECRYPT: {
/* No expected recoverable failures, so no retry list */
goto out;
}
break;
}
}
/* The second step uses len2 and offset2 of the dual_data */
/*
* We would like to call kcf_submit_request() here. But,
* that is not possible as that routine allocates a new
* kcf_areq_node_t request structure, while we need to
* reuse the existing request structure.
*/
switch (pd->pd_prov_type) {
case CRYPTO_SW_PROVIDER:
break;
case CRYPTO_HW_PROVIDER: {
/*
* Set the params for the second step in the
* dual-ops.
*/
/*
* Note that we have to do a taskq_dispatch()
* here as we may be in interrupt context.
*/
TQ_NOSLEEP) == (taskqid_t)0) {
} else {
}
break;
}
}
/*
* We have to release the holds on the request and the provider
* in all cases.
*/
if (error != CRYPTO_QUEUED) {
/* restore, clean up, and invoke the client's callback */
}
}
/*
* Last part of an emulated dual operation.
* Clean up and restore ...
*/
void
{
case KCF_OG_MAC: {
break;
}
case KCF_OG_DECRYPT: {
break;
}
}
/* The submitter used kcf_last_req as its callback */
return;
}
}