physmem.c revision cd64d6e9f50d70599efa5c3e990d0f88efbe4351
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * CDDL HEADER START
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * The contents of this file are subject to the terms of the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Common Development and Distribution License (the "License").
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * You may not use this file except in compliance with the License.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * See the License for the specific language governing permissions
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * and limitations under the License.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * When distributing Covered Code, include this CDDL HEADER in each
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If applicable, add the following below this CDDL HEADER, with the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * fields enclosed by brackets "[]" replaced with your own identifying
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * information: Portions Copyright [yyyy] [name of copyright owner]
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * CDDL HEADER END
cd64d6e9f50d70599efa5c3e990d0f88efbe4351mec * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Use is subject to license terms.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec#pragma ident "%Z%%M% %I% %E% SMI"
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Linked list element hanging off physmem_proc_hash below, which holds all
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * the information for a given segment which has been setup for this process.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * This is a simple linked list as we are assuming that for a given process
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * the setup ioctl will only be called a handful of times. If this assumption
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * changes in the future, a quicker to traverse data structure should be used.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Hash of all of the processes which have setup mappings with the driver with
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * pointers to per process data.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/* Needs to be a power of two for simple hash algorithm */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Lock which protects the pph hash above. To add an element (either a new
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * process or a new segment) the WRITE lock must be held. To traverse the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * list, only a READ lock is needed.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec#define PHYSMEM_HASH(procp) ((int)((((uintptr_t)procp) >> 8) & (PPH_SIZE - 1)))
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Need to keep a reference count of how many processes have the driver
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * open to prevent it from disappearing.
8b464eb836173b92f2b7a65623cd06c8c3c59289mecstatic int physmem_getpage(struct vnode *vp, offset_t off, size_t len,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec uint_t *protp, page_t *pl[], size_t plsz, struct seg *seg, caddr_t addr,
8b464eb836173b92f2b7a65623cd06c8c3c59289mecstatic int physmem_addmap(struct vnode *vp, offset_t off, struct as *as,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec caddr_t addr, size_t len, uchar_t prot, uchar_t maxprot, uint_t flags,
8b464eb836173b92f2b7a65623cd06c8c3c59289mecstatic int physmem_delmap(struct vnode *vp, offset_t off, struct as *as,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec caddr_t addr, size_t len, uint_t prot, uint_t maxprot, uint_t flags,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Removes the current process from the hash if the process has no more
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * physmem segments active.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Add a new entry to the hash for the given process to cache the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * address ranges that it is working on. If this is the first hash
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * item to be added for this process, we will create the head pointer
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * for this process.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Returns 0 on success, ERANGE when the physical address is already in the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * check to make sure a single process does not try to
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * map the same region twice.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (ret == 0) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* Need to check for two threads in sync */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* Dropped the lock so we could use KM_SLEEP */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec newp = kmem_zalloc(sizeof (struct physmem_proc_hash), KM_SLEEP);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Will return the pointer to the physmem_hash struct if the setup routine
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * has previously been called for this memory.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Returns NULL on failure.
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_get_hash(uint64_t req_paddr, size_t len, proc_t *procp)
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Remove the given vnode from the pph hash. If it exists in the hash the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * process still has to be around as the vnode is obviously still around and
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * since it's a physmem vnode, it must be in the hash.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If it is not in the hash that must mean that the setup ioctl failed.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Return 0 in this instance, 1 if it is in the hash.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* synchronize with the map routine */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* not found */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec error = vn_make_ops(name, physmem_vnodeops_template, &physmem_vnodeops);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (error != 0) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec cmn_err(CE_WARN, "physmem_setup_vnops: bad vnode ops template");
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * The guts of the PHYSMEM_SETUP ioctl.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Create a segment in the address space with the specified parameters.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If pspp->user_va is NULL, as_gap will be used to find an appropriate VA.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * We do not do bounds checking on the requested phsycial addresses, if they
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * do not exist in the system, they will not be mappable.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Returns 0 on success with the following error codes on failure:
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * ENOMEM - The VA range requested was already mapped if pspp->user_va is
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * non-NULL or the system was unable to find enough VA space for
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * the desired length if user_va was NULL>
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * EINVAL - The requested PA, VA, or length was not PAGESIZE aligned.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* Sanity checking */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec php = kmem_zalloc(sizeof (struct physmem_hash), KM_SLEEP);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* Need to bump vnode count so that the driver can not be unloaded */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (as_gap(as, len, &uvaddr, &len, AH_LO, NULL) == -1) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* We pick the address for the user */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (ret == 0) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
cd64d6e9f50d70599efa5c3e990d0f88efbe4351mec /* Note that the call to as_unmap will free the vnode */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /*NOTREACHED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * The guts of the PHYSMEM_MAP ioctl.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Map the given PA to the appropriate VA if PHYSMEM_SETUP ioctl has already
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * been called for this PA range.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Returns 0 on success with the following error codes on failure:
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * EPERM - The requested page is long term locked, and thus repeated
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * requests to allocate this page will likely fail.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * EAGAIN - The requested page could not be allocated, but it is believed
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * that future attempts could succeed.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * ENOMEM - There was not enough free memory in the system to safely
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * map the requested page.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * EINVAL - The requested paddr was not PAGESIZE aligned or the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * PHYSMEM_SETUP ioctl was not called for this page.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * ENOENT - The requested page was iniside the kernel cage, and the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * PHYSMEM_CAGE flag was not set.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * EBUSY - The requested page is retired and the PHYSMEM_RETIRE flag
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * was not set.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* Find the vnode for this map request */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec uvaddr = php->ph_base_va + (req_paddr - php->ph_base_pa);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Check to see if page already mapped correctly. This can happen
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * when we failed to capture a page previously and it was captured
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * asynchronously for us. Return success in this case.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * physmem should be responsible for checking for cage
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * and prom pages.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec ret = page_trycapture(pp, 0, flags | CAPTURE_PHYSMEM, curproc);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (ret != 0) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Map the given page into the process's address space if possible.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * We actually only hash the page in on the correct vnode as the page
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * will be mapped via segvn_pagefault.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * returns 0 on success
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * returns 1 if there is no need to map this page anymore (process exited)
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * returns -1 if we failed to map the page.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Check against availrmem to make sure that we're not low on memory.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * We check again here as ASYNC requests do not do this check elsewhere.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * We return 1 as we don't want the page to have the PR_CAPTURE bit
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * set or be on the page capture hash.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If this is an asynchronous request for the current process,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * we can not map the page as it's possible that we are also in the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * process of unmapping the page which could result in a deadlock
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * with the as lock.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (-1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* only return zeroed out pages */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Free the page as there is no longer a valid outstanding
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * request for this page.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * We need to protect against a possible deadlock here where we own
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * the vnode page hash mutex and want to acquire it again as there
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * are locations in the code, where we unlock a page while holding
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * the mutex which can lead to the page being captured and eventually
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * end up here.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (-1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (ret == 0) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (-1);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * The guts of the PHYSMEM_DESTROY ioctl.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * The cookie passed in will provide all of the information needed to
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * free up the address space and physical memory associated with the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * corresponding PHSYMEM_SETUP ioctl.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Returns 0 on success with the following error codes on failure:
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * EINVAL - The cookie supplied is not valid.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec len = ((struct physmem_hash *)(uintptr_t)p_cookie)->ph_seg_len;
8b464eb836173b92f2b7a65623cd06c8c3c59289mec uvaddr = ((struct physmem_hash *)(uintptr_t)p_cookie)->ph_base_va;
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If the page has been hashed into the physmem vnode, then just look it up
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * and return it via pl, otherwise return ENOMEM as the map ioctl has not
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * succeeded on the given page.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_getpage(struct vnode *vp, offset_t off, size_t len, uint_t *protp,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec page_t *pl[], size_t plsz, struct seg *seg, caddr_t addr, enum seg_rw rw,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If the page is in the hash, then we successfully claimed this
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * page earlier, so return it to the caller.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * We can not allow a process mapping /dev/physmem pages to fork as there can
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * only be a single mapping to a /dev/physmem page at a given time. Thus, the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * return of EINVAL when we are not working on our own address space.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Otherwise we return zero as this function is required for normal operation.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_addmap(struct vnode *vp, offset_t off, struct as *as,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec caddr_t addr, size_t len, uchar_t prot, uchar_t maxprot, uint_t flags,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/* Will always get called for removing a whole segment. */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_delmap(struct vnode *vp, offset_t off, struct as *as,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec caddr_t addr, size_t len, uint_t prot, uint_t maxprot, uint_t flags,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Release our hold on the vnode so that the final VN_RELE will
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * call physmem_inactive to clean things up.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Clean up all the pages belonging to this vnode and then free it.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecstatic void
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * Remove the vnode from the hash now, to prevent asynchronous
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * attempts to map into this vnode. This avoids a deadlock
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * where two threads try to get into this logic at the same
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * time and try to map the pages they are destroying into the
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * other's address space.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * If it's not in the hash, just free it.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * At this point in time, no other logic can be adding or removing
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * pages from the vnode, otherwise the v_pages list could be inaccurate.
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * set lckcnt for page_destroy to do availrmem
8b464eb836173b92f2b7a65623cd06c8c3c59289mec * accounting
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* failure to lock should be transient */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec switch (cmd) {
8b464eb836173b92f2b7a65623cd06c8c3c59289mec sizeof (struct physmem_setup_param), 0))
8b464eb836173b92f2b7a65623cd06c8c3c59289mec sizeof (struct physmem_map_param), 0))
8b464eb836173b92f2b7a65623cd06c8c3c59289mec sizeof (uint64_t), 0))
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_open(dev_t *devp, int flag, int otyp, cred_t *credp)
8b464eb836173b92f2b7a65623cd06c8c3c59289mec static int msg_printed = 0;
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* need to make sure we have the right privileges */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec cmn_err(CE_NOTE, "!driver has been opened. This driver may "
8b464eb836173b92f2b7a65623cd06c8c3c59289mec "take out long term locks on pages which may impact "
8b464eb836173b92f2b7a65623cd06c8c3c59289mec "dynamic reconfiguration events");
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mecphysmem_close(dev_t dev, int flag, int otyp, cred_t *credp)
8b464eb836173b92f2b7a65623cd06c8c3c59289mec return (0);
8b464eb836173b92f2b7a65623cd06c8c3c59289mec/*ARGSUSED*/
8b464eb836173b92f2b7a65623cd06c8c3c59289mec if (ddi_create_minor_node(dip, ddi_get_name(dip), S_IFCHR,
8b464eb836173b92f2b7a65623cd06c8c3c59289mec /* Initialize driver specific data */
8b464eb836173b92f2b7a65623cd06c8c3c59289mec for (i = 0; i < PPH_SIZE; i++)
8b464eb836173b92f2b7a65623cd06c8c3c59289mec "physmem driver %I%",