3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * This file and its contents are supplied under the terms of the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * You may only use this file in accordance with the terms of version
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * 1.0 of the CDDL.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * A full copy of the text of the CDDL should have accompanied this
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * source. A copy of the CDDL is also available via the Internet at
6afc9eb26c8915821607cde21b0a32addc165a72Hans Rosenfeld * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved.
2f95345b6f2a0bd2d48718fe10e82e351cb920c6Youzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * blkdev driver for NVMe compliant storage devices
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * This driver was written to conform to version 1.1b of the NVMe specification.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * It may work with newer versions, but that is completely untested and disabled
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * by default.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The driver has only been tested on x86 systems and will not work on big-
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * endian systems without changes to the code accessing registers and data
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * structures used by the hardware.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Interrupt Usage:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The driver will use a FIXED interrupt while configuring the device as the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * specification requires. Later in the attach process it will switch to MSI-X
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * or MSI if supported. The driver wants to have one interrupt vector per CPU,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * but it will work correctly if less are available. Interrupts can be shared
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * by queues, the interrupt handler will iterate through the I/O queue array by
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * steps of n_intr_cnt. Usually only the admin queue will share an interrupt
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * with one I/O queue. The interrupt handler will retrieve completed commands
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * from all queues sharing an interrupt vector and will post them to a taskq
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * for completion processing.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Command Processing:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * available interrupt vector, with the queue length usually much smaller than
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * interrupt vectors will be used.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * hold up to 4096 admin commands.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * From the hardware perspective both queues of a queue pair are independent,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * but they share some driver state: the command array (holding pointers to
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * commands currently being processed by the hardware) and the active command
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * counter. Access to the submission side of a queue pair and the shared state
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * is protected by nq_mutex. The completion side of a queue pair does not need
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * that protection apart from its access to the shared state; it is called only
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * in the interrupt handler which does not run concurrently for the same
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * interrupt vector.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * When a command is submitted to a queue pair the active command counter is
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * incremented and a pointer to the command is stored in the command array. The
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * array index is used as command identifier (CID) in the submission queue
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * wraps around in that time a submission may find the next array slot to still
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * be used by a long-running command. In this case the array is sequentially
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * searched for the next free slot. The length of the command array is the same
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * as the configured queue length.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Namespace Support:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * for each namespace found. Namespaces can have various attributes to support
9cfb65faacc43d80fb0e3cc182d2dae107cdf60cHans Rosenfeld * thin provisioning and protection information. This driver does not support
9cfb65faacc43d80fb0e3cc182d2dae107cdf60cHans Rosenfeld * any of this and ignores namespaces that have these attributes.
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * As of NVMe 1.1 namespaces can have an 64bit Extended Unique Identifier
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * (EUI64). This driver uses the EUI64 if present to generate the devid and
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * passes it to blkdev to use it in the device node names. As this is currently
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * untested namespaces with EUI64 are ignored by default.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Blkdev Interface:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * setup, and splitting of transfers into manageable chunks.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * queues. There is currently no timeout handling of I/O commands.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Blkdev also supports querying device/media information and generating a
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * devid. The driver reports the best block size as determined by the namespace
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * format back to blkdev as physical block size to support partition and block
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * alignment. The devid is either based on the namespace EUI64, if present, or
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * composed using the device vendor ID, model number, serial number, and the
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * namespace ID.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Error Handling:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * either by asynchronous events, or synchronously through command status or
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * requests are posted to allow quicker reception of error information. When an
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * status fields and log information or fault the device, depending on the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * reused and posted to the admin queue again.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * On command completion the command status is checked for errors. In case of
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * status values just cause EIO to be returned.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Command timeouts are currently detected for all admin commands except
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * to be healthy the driver attempts to abort the command. If this fails the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * it. In general admin commands are issued at attach time only. No timeout
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * handling of normal I/O commands is presently done.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * In some cases it may be possible that the ABORT command times out, too. In
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * that case the device is also declared dead and fenced off.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Quiesce / Fast Reboot:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * is still provided which is used to send a shutdown notification to the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Driver Configuration:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The following driver properties can be changed to control some aspects of the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * drivers operation:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * versions or namespaces with EUI64 to be used
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * specific command status as a fatal error leading device faulting
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - io-queue-len: the maximum length of the I/O queues (16-65536)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * posted by the driver
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld * - volatile-write-cache-enable: can be set to 0 to disable the volatile write
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld * - min-phys-block-size: the minimum physical block size to report to blkdev,
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld * which is among other things the basis for ZFS vdev ashift
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - polled I/O support to support kernel core dumping
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - FMA handling of media errors
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - support for querying log pages from user space
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - support for media formatting and hard partitioning into namespaces
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - support for big-endian systems
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - support for fast reboot
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * - support for firmware updates
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * - support for NVMe Subsystem Reset (1.1)
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * - support for Scatter/Gather lists (1.1)
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * - support for Reservations (1.1)
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * - support for power management
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld#error nvme driver needs porting for big-endian platforms
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld/* NVMe spec version supported */
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld/* tunable for admin command timeout in seconds, default is 1s */
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeldstatic volatile int nvme_admin_cmd_timeout = 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_attach(dev_info_t *, ddi_attach_cmd_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_detach(dev_info_t *, ddi_detach_cmd_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_setup_interrupts(nvme_t *, int, int);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_release_interrupts(nvme_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_shutdown(nvme_t *, int, boolean_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic boolean_t nvme_reset(nvme_t *, boolean_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_wakeup_cmd(void *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_async_event_task(void *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_check_unknown_cmd_status(nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_check_vendor_cmd_status(nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_check_integrity_cmd_status(nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_check_specific_cmd_status(nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_check_generic_cmd_status(nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline int nvme_check_cmd_status(nvme_cmd_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void *nvme_get_logpage(nvme_t *, uint8_t, ...);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void *nvme_identify(nvme_t *, uint32_t);
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeldstatic boolean_t nvme_set_features(nvme_t *, uint32_t, uint8_t, uint32_t,
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeldstatic boolean_t nvme_write_cache_set(nvme_t *, boolean_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_set_nqueues(nvme_t *, uint16_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline void nvme_put64(nvme_t *, uintptr_t, uint64_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline void nvme_put32(nvme_t *, uintptr_t, uint32_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline uint64_t nvme_get64(nvme_t *, uintptr_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline uint32_t nvme_get32(nvme_t *, uintptr_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic boolean_t nvme_check_regs_hdl(nvme_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic boolean_t nvme_check_dma_hdl(nvme_dma_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_bd_xfer_done(void *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_bd_driveinfo(void *, bd_drive_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_bd_mediainfo(void *, bd_media_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_bd_write(void *, bd_xfer_t *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *);
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yangstatic int nvme_prp_dma_constructor(void *, void *, int);
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yangstatic void nvme_prp_dma_destructor(void *, void *);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic void nvme_prepare_devid(nvme_t *, uint32_t);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * DMA attributes for queue DMA memory
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * 65536 entries, and an entry can be 64 bytes long.
910f0d12b47aeda4ed059254cc4af0d8c272d0baYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * DMA attributes for transfers using scatter/gather lists
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * length to be a multiple of 16 bytes.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic ddi_device_acc_attr_t nvme_reg_acc_attr = {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline void
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline void
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION);
b6bc2fd4673eae6c96e2aea9e16105dd32a66b7bDan McDonald/* ARGSUSED */
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yangnvme_prp_dma_destructor(void *buf, void *private)
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yangnvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma,
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * indicates a driver bug which should cause a panic.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to get DMA handle, check DMA attributes");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * or the flags are conflicting, which isn't the case here.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to bind DMA memory");
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yangnvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP);
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) !=
b6bc2fd4673eae6c96e2aea9e16105dd32a66b7bDan McDonald/* ARGSUSED */
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yangnvme_prp_dma_constructor(void *buf, void *private, int flags)
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize,
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to get DMA memory for queue");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!got too many cookies for queue DMA");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * slot. If the slot is already occupied advance to the next slot and
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * try again. This can happen for long running commands like async event
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Toggle phase on wrap-around. */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_check_unknown_cmd_status(nvme_cmd_t *cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct,
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_check_integrity_cmd_status(nvme_cmd_t *cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* write fail */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: post ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* read fail */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: post ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_check_generic_cmd_status(nvme_cmd_t *cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Errors indicating a bug in the driver should cause a panic.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Command Opcode */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Field in Command */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Command ID Conflict */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Namespace or Format */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* LBA Out Of Range */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Non-fatal errors, handle gracefully.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Data Transfer Error (DMA) */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: post ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * detailed error information is returned as async event,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * so we pretty much ignore the error here and handle it
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * in the async event handler.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Command Abort Requested. This normally happens only when a
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * command times out.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Command Aborted due to Power Loss Notification */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Command Aborted due to SQ Deletion */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Capacity Exceeded */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Namespace Not Ready */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_check_specific_cmd_status(nvme_cmd_t *cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Completion Queue Invalid */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Queue Identifier */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Max Queue Size Exceeded */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Abort Command Limit Exceeded */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Async Event Request Limit Exceeded */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "async event request limit exceeded in cmd %p",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Interrupt Vector */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Log Page */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Format */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Queue Deletion */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Conflicting Attributes */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Invalid Protection Information */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Write to Read Only Range */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldstatic inline int
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* take a shortcut if everything is alright */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC &&
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (nvme_check_specific_cmd_status(cmd));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (nvme_check_integrity_cmd_status(cmd));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * This functions takes care of cleaning up aborted commands. The command
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * status is checked to catch any fatal errors.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Grab the command mutex. Once we have it we hold the last reference
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * to the command and can safely free it.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Drop the mutex of the aborted command. From this point on
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * we must assume that the abort callback has freed the command.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * the aborted command has completed (aborted or otherwise).
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_admin_cmd failed for ABORT");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!ABORT failed with sct = %x, sc = %x",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Returns B_TRUE if the command completed normally.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Returns B_FALSE if the command timed out and an abort was attempted. The
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * command mutex will be dropped and the command must be considered freed. The
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * freeing of the command is normally done by the abort command callback.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * will be declared dead and FMA will be notified.
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The command timed out. Change the callback to the cleanup function.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Check controller for fatal status, any errors associated with the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * out). If necessary log a warning and call FMA.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command timeout, "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Try to abort the command. The command mutex is released by
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * nvme_abort_cmd().
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * If the abort succeeds it will have freed the aborted command.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * If the abort fails for other reasons we must assume that the
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * command may complete at any time, and the callback will free
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * There is a slight chance that this command completed shortly after
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * the timeout was hit in nvme_wait_cmd() but before the callback was
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * changed. Catch that case here and clean up accordingly.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Check for errors associated with the async request itself. The only
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * command-specific error is "async event limit exceeded", which
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * indicates a programming error in the driver and causes a panic in
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * nvme_check_cmd_status().
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Other possible errors are various scenarios where the async request
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * was aborted, or internal errors in the device. Internal errors are
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * reported to FMA, the command aborts need no special handling here.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!async event request returned failure, sct = %x, "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC &&
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Clear CQE and re-submit the async request. */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to resubmit async event request");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&nvme->n_async_resubmit_failed);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "async event reply: %d", event.b.ae_logpage);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "invalid submission queue");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "invalid doorbell write value");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "device error");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "device error");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: send ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!firmware image load error");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage, -1);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "async event reply: %d", event.b.ae_logpage);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!device reliability compromised");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: send ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!temperature above threshold");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: send ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!spare space below threshold");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: send ereport */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, "
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(error_log, sizeof (nvme_error_log_entry_t) *
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(health_log, sizeof (nvme_health_log_t));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_submit_cmd failed");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The command timed out. An abort command was posted that
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * will take care of the cleanup.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d",
9898bbdb3aee66342820dd5570925a6ad4783d08Hans Rosenfeld getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!too many DMA cookies for GET LOG PAGE");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress;
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_admin_cmd failed for GET LOG PAGE");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!too many DMA cookies for IDENTIFY");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress;
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_admin_cmd failed for IDENTIFY");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE);
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeldnvme_set_features(nvme_t *nvme, uint32_t nsid, uint8_t feature, uint32_t val,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld "!nvme_admin_cmd failed for SET FEATURES");
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld "!SET FEATURES %d failed with sct = %x, sc = %x",
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeldnvme_write_cache_set(nvme_t *nvme, boolean_t enable)
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld if (!nvme_set_features(nvme, 0, NVME_FEAT_WRITE_CACHE, nwc.r, &nwc.r))
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeldnvme_set_nqueues(nvme_t *nvme, uint16_t nqueues)
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld if (!nvme_set_features(nvme, 0, NVME_FEAT_NQUEUES, nq.r, &nq.r)) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Always use the same number of submission and completion queues, and
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * never use more than the requested number of queues.
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress;
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_admin_cmd failed for CREATE CQUEUE");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress;
e8ba2a389f6ca6999ca72dabbe2871e894bf6b67Hans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!nvme_admin_cmd failed for CREATE SQUEUE");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i != 10; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_prepare_devid(nvme_t *nvme, uint32_t nsid)
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * Section 7.7 of the spec describes how to get a unique ID for
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * the controller: the vendor ID, the model name and the serial
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * number shall be unique when combined.
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * If a namespace has no EUI64 we use the above and add the hex
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * namespace ID to get a unique ID for the namespace.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1];
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1];
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0';
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0';
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld nvme->n_ns[nsid - 1].ns_devid = kmem_asprintf("%4X-%s-%s-%X",
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1];
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* Check controller version */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d",
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld nvme->n_version.v_major, nvme->n_version.v_minor);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* retrieve controller configuration */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!NVM command set not supported by hardware");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * page size value.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Maxxfer may be increased after we identified the controller limits.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize;
2f95345b6f2a0bd2d48718fe10e82e351cb920c6Youzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Reset controller if it's still in ready state.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Create the admin queue pair.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!unable to allocate admin qpair");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress;
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Wait for the controller to become ready.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!controller fatal status at init");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * that later when we know the true abort command limit.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL);
bf26ea4b8945ba545fde8b47a2a31ce2d11b82cdHans Rosenfeld * Setup initial interrupt for admin queue.
bf26ea4b8945ba545fde8b47a2a31ce2d11b82cdHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1)
bf26ea4b8945ba545fde8b47a2a31ce2d11b82cdHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1)
bf26ea4b8945ba545fde8b47a2a31ce2d11b82cdHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1)
bf26ea4b8945ba545fde8b47a2a31ce2d11b82cdHans Rosenfeld "!failed to setup initial interrupt");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Post an asynchronous event command to catch errors.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to post async event");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Identify Controller
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to identify controller");
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld * Get Vendor & Product ID
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model));
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0';
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Get controller limits.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit)));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "async-event-limit", nvme->n_async_event_limit);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1;
6afc9eb26c8915821607cde21b0a32addc165a72Hans Rosenfeld * Reinitialize the semaphore with the true abort command limit
6afc9eb26c8915821607cde21b0a32addc165a72Hans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts
6afc9eb26c8915821607cde21b0a32addc165a72Hans Rosenfeld * as only command aborts use the semaphore, and no commands are
6afc9eb26c8915821607cde21b0a32addc165a72Hans Rosenfeld * executed or aborted while we're here.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Chained PRPs are currently unsupported.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * big enough to require chained PRPs.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * larger/smaller than the default.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t)))
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Check for the presence of a Volatile Write Cache. If present,
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld * enable or disable based on the value of the property
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld * volatile-write-cache-enable (default is enabled).
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE;
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld "volatile-write-cache-present",
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld } else if (!nvme_write_cache_set(nvme, nvme->n_write_cache_enabled)) {
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld "!failed to %sable volatile write cache",
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld * Assume the cache is (still) enabled.
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld "volatile-write-cache-enable",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Grab a copy of all mandatory log pages.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * TODO: should go away once user space tool exists to print logs
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_error_log = (nvme_error_log_entry_t *)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Identify Namespaces
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) *
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ns[i].ns_block_count = idns->id_nsize;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size;
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * Get the EUI64 if present. If not present prepare the devid
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * from other device data.
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1))
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld bcopy(idns->id_eui64, nvme->n_ns[i].ns_eui64,
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld if (*(uint64_t *)nvme->n_ns[i].ns_eui64 == 0) {
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id);
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * Until EUI64 support is tested on real hardware we
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * will ignore namespaces with an EUI64. This can
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld * be overriden by setting strict-version=0 in nvme.conf
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Find the LBA format with no metadata and the best relative
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * performance. A value of 3 means "degraded", 0 is best.
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld if (nvme->n_ns[i].ns_best_block_size < nvme->n_min_block_size)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * We currently don't support namespaces that use either:
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - thin provisioning
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * - protection information
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!ignoring namespace %d, unsupported features: "
9cfb65faacc43d80fb0e3cc182d2dae107cdf60cHans Rosenfeld idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Try to set up MSI/MSI-X interrupts.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX))
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to setup MSI/MSI-X interrupts");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Create I/O queue pairs.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to set number of I/O queues to %d", nqueues);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Reallocate I/O queue array
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) *
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * If we got less queues than we asked for we might as well give
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * some of the interrupt vectors back to the system.
34c938c74e6f278ee870d39330b571ffea1b808ePete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to reduce number of interrupts");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Alloc & register I/O queue pairs
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!unable to allocate I/O qpair %d", i);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!unable to create I/O qpair %d", i);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Post more asynchronous events commands to reduce event reporting
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * latency as suggested by the spec.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 1; i != nvme->n_async_event_limit; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to post async event %d", i);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx %
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent);
bf26ea4b8945ba545fde8b47a2a31ce2d11b82cdHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!%s: ddi_intr_get_supported types failed",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* We want at most one interrupt per queue pair. */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i < count; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap);
6afc9eb26c8915821607cde21b0a32addc165a72Hans Rosenfeld for (i = 0; i < count; i++) {
75b41617efad806d8ab3d1866425189c0b0bc6aaHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1);
75b41617efad806d8ab3d1866425189c0b0bc6aaHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ?
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld DDI_PROP_DONTPASS, "volatile-write-cache-enable", 1) != 0 ?
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) {
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld dev_err(dip, CE_WARN, "!min-phys-block-size %s, "
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld "using default %d", ISP2(nvme->n_min_block_size) ?
6801591e1fea06222c50622c468541c82c67cdb4Hans Rosenfeld nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Setup FMA support.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * The spec defines several register sets. Only the controller
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * registers (set 1) are currently used.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE ||
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1");
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Create taskq for command completion.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq");
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang * Create PRP DMA cache
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache",
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t),
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Attach the blkdev driver for each namespace.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i],
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to get blkdev handle for namespace %d", i);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld "!failed to attach blkdev handle for namespace %d",
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* attach successful anyway so that FMA can retire the device */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) *
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) {
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* TODO: send destroy queue commands */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) *
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * We currently don't support chained PRPs and set up our DMA
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * attributes to reflect that. If we still get an I/O request
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * that needs a chained PRP something is very wrong.
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP);
8834f7ac63a18ac48dfacf20506346d82b04fc14Youzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ?
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_bd_driveinfo(void *arg, bd_drive_t *drive)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * blkdev maintains one queue size per instance (namespace),
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * but all namespace share the I/O queues.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues,
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * or change blkdev to handle EAGAIN
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld * attributes specified to bd_alloc_handle.
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld bcopy(ns->ns_eui64, drive->d_eui64, sizeof (drive->d_eui64));
bef9e21af8ac4992b7dfef56530c7b0ed9184096Hans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model);
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor);
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld drive->d_product_len = strlen(nvme->n_product);
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial);
510a68476ba6e33759b7603130d76db4cec783d1Hans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev);
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_bd_mediainfo(void *arg, bd_media_t *media)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld /* No polling for now */
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1;
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd)
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE));
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld * If the volatile write cache is not present or not enabled the FLUSH
d148d46e69709052e6d92205c98874f3e7cfd4c0Hans Rosenfeld * command is a no-op, so we can take a shortcut here.
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH));
3c9168fa8e9c30d55b3aa2fde74bd7da46df53f5Hans Rosenfeldnvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid)
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld return (ddi_devid_init(devinfo, DEVID_SCSI3_WWN,
24979ca36afa68d08e082148fcbf4c5dc73f7849Hans Rosenfeld sizeof (ns->ns_eui64), ns->ns_eui64, devid));