targets/av1394/av1394_isoch_xmit.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * av1394 isochronous transmit module
 */
#include <sys/1394/targets/av1394/av1394_impl.h>

static int  av1394_it_start_common(av1394_ic_t *);

/* configuration routines */
static void av1394_it_cleanup(av1394_ic_t *, int);
static int  av1394_it_bld_ixl(av1394_ic_t *);
static void av1394_it_destroy_ixl(av1394_ic_t *);
static int  av1394_it_ixl_bld_data(av1394_ic_t *);
static void av1394_it_ixl_destroy_data(av1394_ic_t *);
static av1394_it_ixl_buf_t *av1394_it_ixl_bld_buf(av1394_ic_t *, int, int,
        off_t, int, int);
static void av1394_it_ixl_complete_buf(av1394_it_ixl_buf_t *,
        av1394_it_ixl_empty_cip_t *);
static void av1394_it_ixl_complete_buf2(av1394_it_ixl_buf_t *,
        av1394_it_ixl_buf_t *);
static av1394_it_ixl_empty_cip_t *av1394_it_ixl_bld_empty_cip(av1394_ic_t *,
        int);
static void av1394_it_ixl_complete_empty_cip(av1394_it_ixl_empty_cip_t *,
        av1394_it_ixl_buf_t *);
static void av1394_it_ixl_bld_begin(av1394_ic_t *);
static void av1394_it_ixl_begin_update_pkts(av1394_ic_t *,
        av1394_it_ixl_buf_t *);
static int  av1394_it_alloc_isoch_dma(av1394_ic_t *);
static void av1394_it_free_isoch_dma(av1394_ic_t *);
static void av1394_it_dma_sync_frames(av1394_ic_t *, int, int);

/* callbacks */
static void av1394_it_ixl_begin_cb(opaque_t, struct ixl1394_callback *);
static void av1394_it_ixl_buf_cb(opaque_t, struct ixl1394_callback *);
static void av1394_it_ixl_eof_cb(av1394_it_ixl_buf_t *bp);
static int  av1394_it_underrun_resume(av1394_ic_t *);
static void av1394_it_dma_stopped_cb(t1394_isoch_dma_handle_t,
        opaque_t, id1394_isoch_dma_stopped_t);

/* data transfer routines */
static int  av1394_it_add_frames(av1394_ic_t *, int, int);
static int  av1394_it_wait_frames(av1394_ic_t *, int *, int *, int *);

static void av1394_it_update_frame_syt(av1394_ic_t *, int, int, uint16_t);
static uint16_t av1394_it_ts_cyc2syt(uint16_t);
static uint16_t av1394_it_ts_syt_inc(uint16_t, uint16_t);

static void av1394_it_kcopyin(av1394_ic_t *, void *, size_t);
static int  av1394_it_copyin(av1394_ic_t *, struct uio *, int *, int);
static boolean_t av1394_it_is_dv_frame_start(caddr_t);
static void av1394_it_reset_frame_syt(av1394_ic_t *, int);

/* tunables */
int av1394_it_hiwat_sub = 2;
int av1394_it_lowat = 3;
int av1394_it_start_thre = 3;   /* xmit start threshold */
int av1394_it_syt_off = 3;  /* SYT offset in cycles */
int av1394_it_dump_ixl = 0;

#define AV1394_TNF_ENTER(func)  \
    TNF_PROBE_0_DEBUG(func##_enter, AV1394_TNF_ISOCH_STACK, "");

#define AV1394_TNF_EXIT(func)   \
    TNF_PROBE_0_DEBUG(func##_exit, AV1394_TNF_ISOCH_STACK, "");

int
av1394_it_init(av1394_ic_t *icp, int *error)
{
    av1394_it_t *itp = &icp->ic_it;
    av1394_isoch_pool_t *pool = &itp->it_data_pool;
    int     nframes;

    AV1394_TNF_ENTER(av1394_it_init);

    nframes = av1394_ic_alloc_pool(pool, icp->ic_framesz, icp->ic_nframes,
        AV1394_IT_NFRAMES_MIN);
    if (nframes == 0) {
        *error = IEC61883_ERR_NOMEM;
        AV1394_TNF_EXIT(av1394_it_init);
        return (EINVAL);
    }
    mutex_enter(&icp->ic_mutex);
    icp->ic_nframes = nframes;
    itp->it_hiwat = nframes - av1394_it_hiwat_sub;
    itp->it_lowat = av1394_it_lowat;
    itp->it_start_thre = av1394_it_start_thre;
    itp->it_nempty = icp->ic_nframes;
    itp->it_last_full = icp->ic_nframes - 1;

    if (av1394_ic_dma_setup(icp, pool) != DDI_SUCCESS) {
        mutex_exit(&icp->ic_mutex);
        *error = IEC61883_ERR_NOMEM;
        av1394_it_cleanup(icp, 1);
        AV1394_TNF_EXIT(av1394_it_init);
        return (EINVAL);
    }

    if (av1394_it_bld_ixl(icp) != DDI_SUCCESS) {
        mutex_exit(&icp->ic_mutex);
        *error = IEC61883_ERR_NOMEM;
        av1394_it_cleanup(icp, 2);
        AV1394_TNF_EXIT(av1394_it_init);
        return (EINVAL);
    }
    mutex_exit(&icp->ic_mutex);

    if (av1394_it_alloc_isoch_dma(icp) != DDI_SUCCESS) {
        *error = IEC61883_ERR_NOMEM;
        av1394_it_cleanup(icp, 3);
        AV1394_TNF_EXIT(av1394_it_init);
        return (EINVAL);
    }

    AV1394_TNF_EXIT(av1394_it_init);
    return (0);
}

void
av1394_it_fini(av1394_ic_t *icp)
{
    AV1394_TNF_ENTER(av1394_it_fini);

    av1394_it_cleanup(icp, AV1394_CLEANUP_LEVEL_MAX);

    AV1394_TNF_ENTER(av1394_it_fini);
}

int
av1394_it_start(av1394_ic_t *icp)
{
    av1394_it_t *itp = &icp->ic_it;
    int     ret = 0;

    AV1394_TNF_ENTER(av1394_it_start);

    mutex_enter(&icp->ic_mutex);
    ASSERT(icp->ic_state == AV1394_IC_IDLE);

    /* should be enough full frames to be able to start */
    if (itp->it_nfull >= itp->it_start_thre) {
        ret = av1394_it_start_common(icp);
    }
    mutex_exit(&icp->ic_mutex);

    AV1394_TNF_EXIT(av1394_it_start);
    return (ret);
}

static int
av1394_it_start_common(av1394_ic_t *icp)
{
    av1394_inst_t   *avp = icp->ic_avp;
    id1394_isoch_dma_ctrlinfo_t idma_ctrlinfo = { 0 };
    int     result;
    int     err;
    int     ret = 0;

    ASSERT(icp->ic_state == AV1394_IC_IDLE);

    err = t1394_start_isoch_dma(avp->av_t1394_hdl, icp->ic_isoch_hdl,
        &idma_ctrlinfo, 0, &result);
    if (err == DDI_SUCCESS) {
        icp->ic_state = AV1394_IC_DMA;
    } else {
        TNF_PROBE_1(av1394_it_start_common_error,
            AV1394_TNF_ISOCH_ERROR, "", tnf_int, result, result);
        ret = EIO;
    }

    return (ret);
}


int
av1394_it_stop(av1394_ic_t *icp)
{
    av1394_inst_t   *avp = icp->ic_avp;
    av1394_it_t *itp = &icp->ic_it;

    AV1394_TNF_ENTER(av1394_it_stop);

    mutex_enter(&icp->ic_mutex);
    if (icp->ic_state != AV1394_IC_IDLE) {
        mutex_exit(&icp->ic_mutex);
        t1394_stop_isoch_dma(avp->av_t1394_hdl, icp->ic_isoch_hdl, 0);
        mutex_enter(&icp->ic_mutex);

        icp->ic_state = AV1394_IC_IDLE;
        itp->it_nempty = icp->ic_nframes;
        itp->it_first_empty = 0;
        itp->it_last_full = icp->ic_nframes - 1;
        itp->it_nfull = 0;
    }
    mutex_exit(&icp->ic_mutex);

    AV1394_TNF_EXIT(av1394_it_stop);
    return (0);
}

int
av1394_it_xmit(av1394_ic_t *icp, iec61883_xmit_t *xmit)
{
    av1394_it_t *itp = &icp->ic_it;
    int     ret = 0;
    int     idx, cnt;

    idx = xmit->tx_xfer.xf_full_idx;
    cnt = xmit->tx_xfer.xf_full_cnt;

    mutex_enter(&icp->ic_mutex);
    /* check arguments */
    if ((idx < 0) || (cnt < 0) || (cnt > itp->it_nempty)) {
        mutex_exit(&icp->ic_mutex);
        TNF_PROBE_2(av1394_it_xmit_error_args, AV1394_TNF_ISOCH_ERROR,
            "", tnf_int, idx, idx, tnf_int, cnt, cnt);
        return (EINVAL);
    }

    /* add full frames to the pool */
    if (cnt > 0) {
        if ((ret = av1394_it_add_frames(icp, idx, cnt)) != 0) {
            mutex_exit(&icp->ic_mutex);
            return (ret);
        }
    }

    if ((icp->ic_state == AV1394_IC_IDLE) &&
        (itp->it_nfull >= itp->it_start_thre)) {
        if ((ret = av1394_it_start_common(icp)) != 0) {
            mutex_exit(&icp->ic_mutex);
            return (ret);
        }
    }

    /* wait for new empty frames */
    ret = av1394_it_wait_frames(icp, &xmit->tx_xfer.xf_empty_idx,
        &xmit->tx_xfer.xf_empty_cnt, &xmit->tx_miss_cnt);
    mutex_exit(&icp->ic_mutex);

    return (ret);
}

int
av1394_it_write(av1394_ic_t *icp, struct uio *uiop)
{
    av1394_inst_t   *avp = icp->ic_avp;
    av1394_it_t *itp = &icp->ic_it;
    av1394_isoch_autoxmit_t *axp = &avp->av_i.i_autoxmit;
    int     dv;
    int     ret = 0;
    int     full_cnt;
    int     miss_cnt;

    AV1394_TNF_ENTER(av1394_it_write);

    mutex_enter(&icp->ic_mutex);
    dv = (axp->ax_fmt & AV1394_ISOCH_AUTOXMIT_DV);

    while (uiop->uio_resid > 0) {
        /* must have at least one empty frame */
        if (itp->it_write_cnt == 0) {
            ret = av1394_it_wait_frames(icp, &itp->it_write_idx,
                &itp->it_write_cnt, &miss_cnt);
            if (ret != 0) {
                break;
            }
        }

        /* copyin as much data as we can */
        if (axp->ax_copy_ciph) {
            ASSERT(itp->it_write_off == 0);
            av1394_it_kcopyin(icp, axp->ax_ciph, AV1394_CIPSZ);
            axp->ax_copy_ciph = B_FALSE;
        }
        if ((ret = av1394_it_copyin(icp, uiop, &full_cnt, dv)) != 0) {
            break;
        }

        /* add full frames to the pool */
        if (full_cnt > 0) {
            ret = av1394_it_add_frames(icp,
                itp->it_write_idx, full_cnt);
            if (ret != 0) {
                break;
            }
            itp->it_write_idx += full_cnt;
            itp->it_write_idx %= icp->ic_nframes;
        }

        /* start xfer if not already */
        if ((icp->ic_state == AV1394_IC_IDLE) &&
            (itp->it_nfull >= itp->it_start_thre)) {
            if ((ret = av1394_it_start_common(icp)) != 0) {
                mutex_exit(&icp->ic_mutex);
                AV1394_TNF_EXIT(av1394_it_write);
                return (ret);
            }
        }
    }
    mutex_exit(&icp->ic_mutex);

    AV1394_TNF_EXIT(av1394_it_write);
    return (ret);
}

/*
 *
 * --- configuration routines
 *
 */
static void
av1394_it_cleanup(av1394_ic_t *icp, int level)
{
    av1394_isoch_pool_t *pool = &icp->ic_it.it_data_pool;

    ASSERT((level > 0) && (level <= AV1394_CLEANUP_LEVEL_MAX));

    switch (level) {
    default:
        av1394_it_free_isoch_dma(icp);
        /* FALLTHRU */
    case 3:
        av1394_it_destroy_ixl(icp);
        /* FALLTHRU */
    case 2:
        av1394_ic_dma_cleanup(icp, pool);
        /* FALLTHRU */
    case 1:
        av1394_ic_free_pool(pool);
        /* FALLTHRU */
    }
}

/*
 * av1394_it_bld_ixl()
 *    Build an IXL chain out of several blocks.
 */
static int
av1394_it_bld_ixl(av1394_ic_t *icp)
{
    av1394_it_t *itp = &icp->ic_it;
    int     ret;

    AV1394_TNF_ENTER(av1394_it_bld_ixl);

    /* data block */
    if ((ret = av1394_it_ixl_bld_data(icp)) != DDI_SUCCESS) {
        AV1394_TNF_EXIT(av1394_it_bld_ixl);
        return (ret);
    }

    /* begin block */
    if (icp->ic_param.cp_ts_mode != IEC61883_TS_NONE) {
        av1394_it_ixl_bld_begin(icp);

        itp->it_ixlp = (ixl1394_command_t *)&itp->it_ixl_begin;
    } else {
        itp->it_ixlp = (ixl1394_command_t *)
            &((av1394_it_ixl_buf_t *)itp->it_ixl_data)->tb_label;
    }

    if (av1394_it_dump_ixl) {
        av1394_ic_ixl_dump(itp->it_ixlp);
    }

    AV1394_TNF_EXIT(av1394_it_bld_ixl);
    return (ret);
}

static void
av1394_it_destroy_ixl(av1394_ic_t *icp)
{
    av1394_it_t *itp = &icp->ic_it;

    av1394_it_ixl_destroy_data(icp);
    itp->it_ixlp = NULL;
}

/*
 * build data transmit part of the IXL chain
 */
static int
av1394_it_ixl_bld_data(av1394_ic_t *icp)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_isoch_pool_t *pool = &itp->it_data_pool;
    int         total = 0;  /* # of pkts in the chain */
    int         nfull = 0;  /* # of full CIPs in a series */
    int         framenum = -1;  /* frame number */
    int         bufsz_max;  /* max buffer size in pkts */
    int         segnum = 0; /* current segment number */
    int         segsz;      /* segment size in pkts */
    off_t           segoff = 0; /* segment offset in pkts */
    av1394_it_ixl_empty_cip_t *ep = NULL;   /* last empty CIP */
    av1394_it_ixl_buf_t *bp = NULL; /* last data buffer */
    av1394_it_ixl_buf_t *prevbp = NULL;
    int         a, n, d;    /* N/D algorithm variables */
    int         type, ptype;    /* current and prev CIP type */
    int         tb_flags;

    itp->it_frame_info = kmem_zalloc(icp->ic_nframes *
        sizeof (av1394_it_frame_info_t), KM_SLEEP);

    bufsz_max = AV1394_IXL_BUFSZ_MAX / icp->ic_pktsz;
    n = icp->ic_param.cp_n;
    d = icp->ic_param.cp_d;
    /*
     * following assert guarantees no more than one empty CIP in a row,
     * i.e. empty CIPs account for <=50% of all packets.
     * this should be ensured by ioctl argument validation.
     */
    ASSERT((n == 0) || (d / n > 1));
    /*
     * build the chain. it is hard to precalculate amount of memory
     * needed for the entire chain, so we simply allocate as we go.
     */
    ptype = AV1394_CIP_EMPTY;
    segsz = pool->ip_seg[0].is_size / icp->ic_pktsz;
    a = n;
    while (total < icp->ic_nframes * icp->ic_npkts) {
        /* insert empty CIPs using N/D algorithm */
        a += n;
        if (a > d) {
            a -= d;
            type = AV1394_CIP_EMPTY;
        } else {
            type = AV1394_CIP_FULL;
            nfull++;
        }

        /*
         * merge series of full packets into single SEND_BUF commands.
         * a series can be terminated by:
         *  - an empty CIP;
         *  - series buffer size reached maximum;
         *  - end of isoch segment;
         *  - end of frame (which is always at the end of segment);
         */
        if (((type == AV1394_CIP_EMPTY) || (segoff + nfull == segsz) ||
            (nfull == bufsz_max)) && (nfull > 0)) {

            /* build buffer block */
            prevbp = bp;
            tb_flags = 0;
            if (type == AV1394_CIP_EMPTY) {
                tb_flags |= AV1394_IT_IXL_BUF_NEXT_EMPTY;
            }
            if (total % icp->ic_npkts == 0) {
                tb_flags |= AV1394_IT_IXL_BUF_SOF;
                framenum++;
            }
            if ((total + nfull) % icp->ic_npkts == 0) {
                tb_flags |= AV1394_IT_IXL_BUF_EOF;
            }
            bp = av1394_it_ixl_bld_buf(icp, nfull, segnum, segoff,
                tb_flags, framenum);

            if (itp->it_ixl_data == NULL) {
                itp->it_ixl_data = &bp->tb_common;
            }

            /* complete previous empty CIP or a buffer */
            if (ep) {
                av1394_it_ixl_complete_empty_cip(ep, bp);
                ep = NULL;
            } else if (prevbp) {
                av1394_it_ixl_complete_buf2(prevbp, bp);
            }

            /* if current segment is used up, pick next one */
            segoff += nfull;
            if (segoff == segsz) {
                if (++segnum < pool->ip_nsegs) {
                    segsz = pool->ip_seg[segnum].is_size /
                        icp->ic_pktsz;
                }
                segoff = 0;
            }

            total += nfull;
            nfull = 0;
        }
        /* insert an empty packet if needed */
        if (type == AV1394_CIP_EMPTY) {
            ep = av1394_it_ixl_bld_empty_cip(icp, framenum);
            av1394_it_ixl_complete_buf(bp, ep);
        }
        ptype = type;
    }
    ASSERT(nfull == 0);

    /* last packet must be an empty CIP, except when n == 0 */
    if (n != 0) {
        if (ptype != AV1394_CIP_EMPTY) {
            ep = av1394_it_ixl_bld_empty_cip(icp, framenum);
            av1394_it_ixl_complete_buf(bp, ep);
        }
        av1394_it_ixl_complete_empty_cip(ep,
            (av1394_it_ixl_buf_t *)itp->it_ixl_data);
        ep->te_jump.next_ixlp = NULL;
        ep->te_common.tc_next = NULL;
    } else {
        bp->tb_jump.label = (ixl1394_command_t *)
            &(((av1394_it_ixl_buf_t *)itp->it_ixl_data)->tb_label);
    }

    return (DDI_SUCCESS);
}

static void
av1394_it_ixl_destroy_data(av1394_ic_t *icp)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_it_ixl_common_t  *cmd, *cmd_next;

    for (cmd = itp->it_ixl_data; cmd != NULL; cmd = cmd_next) {
        cmd_next = cmd->tc_next;
        kmem_free(cmd, cmd->tc_size);
    }
    itp->it_ixl_data = NULL;

    kmem_free(itp->it_frame_info,
        icp->ic_nframes * sizeof (av1394_it_frame_info_t));
}

static av1394_it_ixl_buf_t *
av1394_it_ixl_bld_buf(av1394_ic_t *icp, int cnt, int segnum, off_t off,
        int flags, int framenum)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_isoch_seg_t  *isp = &itp->it_data_pool.ip_seg[segnum];
    av1394_it_ixl_buf_t *bp;
    int         pktsz = icp->ic_pktsz;

    bp = kmem_zalloc(sizeof (av1394_it_ixl_buf_t), KM_SLEEP);
    bp->tb_common.tc_size = sizeof (av1394_it_ixl_buf_t);
    /* tc_next later */
    bp->tb_flags = flags;
    bp->tb_framenum = framenum;
    bp->tb_icp = icp;

    bp->tb_label.ixl_opcode = IXL1394_OP_LABEL;
    bp->tb_label.next_ixlp = (ixl1394_command_t *)&bp->tb_buf;

    bp->tb_buf.ixl_opcode = IXL1394_OP_SEND_BUF;
    bp->tb_buf.pkt_size = pktsz;
    bp->tb_buf.size = cnt * pktsz;
    bp->tb_buf.ixl_buf._dmac_ll =
        isp->is_dma_cookie[0].dmac_laddress + off * pktsz;
    bp->tb_buf.mem_bufp = isp->is_kaddr + off * pktsz;

    if (flags & AV1394_IT_IXL_BUF_EOF) {
        bp->tb_buf.next_ixlp = (ixl1394_command_t *)&bp->tb_store_ts;

        bp->tb_store_ts.ixl_opcode = IXL1394_OP_STORE_TIMESTAMP;
        bp->tb_store_ts.next_ixlp = (ixl1394_command_t *)&bp->tb_cb;

        bp->tb_cb.ixl_opcode = IXL1394_OP_CALLBACK;
        bp->tb_cb.callback = av1394_it_ixl_buf_cb;
        bp->tb_cb.callback_arg = bp;
        bp->tb_cb.next_ixlp = (ixl1394_command_t *)&bp->tb_jump;

        bp->tb_jump.ixl_opcode = IXL1394_OP_JUMP_U;
    } else {
        bp->tb_buf.next_ixlp = (ixl1394_command_t *)&bp->tb_jump;

        bp->tb_jump.ixl_opcode = IXL1394_OP_JUMP;
    }
    /*
     * jump label and next_ixlp later.
     * unset fields will be set in av1394_it_ixl_complete_buf()
     *
     * save additional frame info
     */
    if (flags & AV1394_IT_IXL_BUF_SOF) {
        itp->it_frame_info[framenum].fi_first_buf = bp;
        itp->it_frame_info[framenum].fi_ts_off = bp->tb_buf.mem_bufp +
            AV1394_TS_MODE_GET_OFF(icp->ic_param.cp_ts_mode);
    } else if (flags & AV1394_IT_IXL_BUF_EOF) {
        itp->it_frame_info[framenum].fi_last_buf = bp;
    }
    itp->it_frame_info[framenum].fi_ncycs += cnt;

    return (bp);
}

static void
av1394_it_ixl_complete_buf(av1394_it_ixl_buf_t *bp,
    av1394_it_ixl_empty_cip_t *ep)
{
    bp->tb_common.tc_next = &ep->te_common;
    bp->tb_jump.label = bp->tb_jump.next_ixlp =
        (ixl1394_command_t *)&ep->te_label;
}

static void
av1394_it_ixl_complete_buf2(av1394_it_ixl_buf_t *bp,
    av1394_it_ixl_buf_t *nextbp)
{
    bp->tb_common.tc_next = &nextbp->tb_common;
    bp->tb_jump.label = bp->tb_jump.next_ixlp =
        (ixl1394_command_t *)&nextbp->tb_label;
}

static av1394_it_ixl_empty_cip_t *
av1394_it_ixl_bld_empty_cip(av1394_ic_t *icp, int framenum)
{
    av1394_it_t *itp = &icp->ic_it;
    av1394_it_ixl_empty_cip_t *ep;

    ep = kmem_zalloc(sizeof (av1394_it_ixl_empty_cip_t), KM_SLEEP);
    ep->te_common.tc_size = sizeof (av1394_it_ixl_empty_cip_t);
    /* tc_next later */

    ep->te_label.ixl_opcode = IXL1394_OP_LABEL;
    ep->te_label.next_ixlp = (ixl1394_command_t *)&ep->te_pkt;

    ep->te_pkt.ixl_opcode = IXL1394_OP_SEND_PKT_ST;
    ep->te_pkt.size = AV1394_CIPSZ;
    /* ixl_buf and mem_bufp later */
    ep->te_pkt.next_ixlp = (ixl1394_command_t *)&ep->te_jump;

    ep->te_jump.ixl_opcode = IXL1394_OP_JUMP;
    /*
     * label and next_ixlp later.
     * unset fields will be set in av1394_it_ixl_complete_empty_cip()
     */

    itp->it_frame_info[framenum].fi_ncycs++;

    return (ep);
}

/*
 * empty CIP packet contains CIP header of the next packet,
 * so we just point to the same address as the next packet's header
 */
static void
av1394_it_ixl_complete_empty_cip(av1394_it_ixl_empty_cip_t *ep,
    av1394_it_ixl_buf_t *bp)
{
    ep->te_common.tc_next = &bp->tb_common;

    ep->te_pkt.ixl_buf._dmac_ll = bp->tb_buf.ixl_buf._dmac_ll;
    ep->te_pkt.mem_bufp = bp->tb_buf.mem_bufp;

    ep->te_jump.label = ep->te_jump.next_ixlp =
        (ixl1394_command_t *)&bp->tb_label;
}

static void
av1394_it_ixl_bld_begin(av1394_ic_t *icp)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_it_ixl_buf_t *bp = (av1394_it_ixl_buf_t *)itp->it_ixl_data;
    av1394_it_ixl_begin_t   *bep = &itp->it_ixl_begin;
    int         i;

    bep->be_label.ixl_opcode = IXL1394_OP_LABEL;
    bep->be_label.next_ixlp = (ixl1394_command_t *)&bep->be_empty_pre;

    bep->be_empty_pre.ixl_opcode = IXL1394_OP_SEND_PKT_ST;
    bep->be_empty_pre.size = AV1394_CIPSZ;
    bep->be_empty_pre.ixl_buf._dmac_ll = bp->tb_buf.ixl_buf._dmac_ll;
    bep->be_empty_pre.mem_bufp = bp->tb_buf.mem_bufp;
    bep->be_empty_pre.next_ixlp = (ixl1394_command_t *)&bep->be_store_ts;

    bep->be_store_ts.ixl_opcode = IXL1394_OP_STORE_TIMESTAMP;
    bep->be_store_ts.next_ixlp = (ixl1394_command_t *)&bep->be_cb;

    bep->be_cb.ixl_opcode = IXL1394_OP_CALLBACK;
    bep->be_cb.callback = av1394_it_ixl_begin_cb;
    bep->be_cb.callback_arg = &bep->be_store_ts.timestamp;
    bep->be_cb.next_ixlp = (ixl1394_command_t *)&bep->be_empty_post[0];

    for (i = 0; i < AV1394_IT_IXL_BEGIN_NPOST; i++) {
        bep->be_empty_post[i].ixl_opcode = IXL1394_OP_SEND_PKT_ST;
        bep->be_empty_post[i].size = AV1394_CIPSZ;
        bep->be_empty_post[i].ixl_buf._dmac_ll =
            bp->tb_buf.ixl_buf._dmac_ll;
        bep->be_empty_post[i].mem_bufp = bp->tb_buf.mem_bufp;
        bep->be_empty_post[i].next_ixlp =
            (ixl1394_command_t *)&bep->be_empty_post[i + 1];
    }
    bep->be_empty_post[AV1394_IT_IXL_BEGIN_NPOST - 1].next_ixlp =
        (ixl1394_command_t *)&bep->be_jump;

    bep->be_jump.ixl_opcode = IXL1394_OP_JUMP_U;
    bep->be_jump.label = (ixl1394_command_t *)&bp->tb_label;
    bep->be_jump.next_ixlp = (ixl1394_command_t *)&bp->tb_label;
}

static void
av1394_it_ixl_begin_update_pkts(av1394_ic_t *icp, av1394_it_ixl_buf_t *bp)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_it_ixl_begin_t   *bep = &itp->it_ixl_begin;
    int         i;

    for (i = 0; i < AV1394_IT_IXL_BEGIN_NPOST; i++) {
        bep->be_empty_post[i].ixl_buf._dmac_ll =
            bp->tb_buf.ixl_buf._dmac_ll;
        bep->be_empty_post[i].mem_bufp = bp->tb_buf.mem_bufp;
    }
}

static int
av1394_it_alloc_isoch_dma(av1394_ic_t *icp)
{
    av1394_inst_t       *avp = icp->ic_avp;
    av1394_it_t     *itp = &icp->ic_it;
    id1394_isoch_dmainfo_t  di;
    int         result;
    int         ret;

    AV1394_TNF_ENTER(av1394_it_alloc_isoch_dma);

    di.ixlp = itp->it_ixlp;
    di.channel_num = icp->ic_num;
    di.idma_options = ID1394_TALK;
    di.it_speed = icp->ic_param.cp_bus_speed;
    /*
     * XXX this should really be IXL1394_SKIP_TO_NEXT,
     * but it can't be used yet due to the Framework bug
     */
    di.it_default_skip = IXL1394_SKIP_TO_SELF;
    di.default_tag = 1;
    di.default_sync = 0;
    di.global_callback_arg = icp;
    di.isoch_dma_stopped = av1394_it_dma_stopped_cb;
    di.idma_evt_arg = icp;

    if ((ret = t1394_alloc_isoch_dma(avp->av_t1394_hdl, &di, 0,
        &icp->ic_isoch_hdl, &result)) != DDI_SUCCESS) {
        TNF_PROBE_1(av1394_it_alloc_isoch_dma_error,
            AV1394_TNF_ISOCH_ERROR, "", tnf_int, result, result);
    }

    AV1394_TNF_EXIT(av1394_it_alloc_isoch_dma);
    return (ret);
}

static void
av1394_it_free_isoch_dma(av1394_ic_t *icp)
{
    av1394_inst_t       *avp = icp->ic_avp;

    AV1394_TNF_ENTER(av1394_it_free_isoch_rsrc);

    t1394_free_isoch_dma(avp->av_t1394_hdl, 0, &icp->ic_isoch_hdl);

    AV1394_TNF_EXIT(av1394_it_free_isoch_rsrc);
}

static void
av1394_it_dma_sync_frames(av1394_ic_t *icp, int idx, int cnt)
{
    av1394_ic_dma_sync_frames(icp, idx, cnt,
        &icp->ic_it.it_data_pool, DDI_DMA_SYNC_FORDEV);
}

/*
 *
 * --- callbacks
 *
 */
static void
av1394_it_ixl_begin_cb(opaque_t arg, struct ixl1394_callback *cb)
{
    av1394_ic_t *icp = arg;
    av1394_it_t *itp = &icp->ic_it;
    uint16_t    *cycp = cb->callback_arg; /* cycle timestamp pointer */
    uint16_t    syt;
    int     first;

    AV1394_TNF_ENTER(av1394_it_ixl_begin_cb);

    mutex_enter(&icp->ic_mutex);
    /* save initial timestamp value */
    itp->it_ts_init.ts_syt = av1394_it_ts_cyc2syt(*cycp);

    /*
     * update frame timestamps if needed
     */
    if ((itp->it_nfull <= 0) ||
        (AV1394_TS_MODE_GET_SIZE(icp->ic_param.cp_ts_mode) == 0)) {
        mutex_exit(&icp->ic_mutex);
        AV1394_TNF_EXIT(av1394_it_ixl_begin_cb);
        return;
    }
    ASSERT(itp->it_nfull <= icp->ic_nframes);

    syt = av1394_it_ts_syt_inc(itp->it_ts_init.ts_syt,
        AV1394_IT_IXL_BEGIN_NPOST + av1394_it_syt_off);
    first = (itp->it_last_full + icp->ic_nframes - itp->it_nfull + 1) %
        icp->ic_nframes;
    av1394_it_update_frame_syt(icp, first, itp->it_nfull, syt);
    mutex_exit(&icp->ic_mutex);

    AV1394_TNF_EXIT(av1394_it_ixl_begin_cb);
}

/*ARGSUSED*/
static void
av1394_it_ixl_buf_cb(opaque_t arg, struct ixl1394_callback *cb)
{
    av1394_it_ixl_buf_t *bp = cb->callback_arg;

    AV1394_TNF_ENTER(av1394_it_ixl_buf_cb);
    if (bp->tb_flags & AV1394_IT_IXL_BUF_EOF) {
        av1394_it_ixl_eof_cb(bp);
    }
    AV1394_TNF_EXIT(av1394_it_ixl_buf_cb);
}

static void
av1394_it_ixl_eof_cb(av1394_it_ixl_buf_t *bp)
{
    av1394_ic_t *icp = bp->tb_icp;
    av1394_isoch_t  *ip = &icp->ic_avp->av_i;
    av1394_it_t *itp = &icp->ic_it;

    AV1394_TNF_ENTER(av1394_it_ixl_eof_cb);

    mutex_enter(&ip->i_mutex);
    mutex_enter(&icp->ic_mutex);
    if (itp->it_nempty < icp->ic_nframes) {
        itp->it_nempty++;
        itp->it_nfull--;
        cv_signal(&icp->ic_xfer_cv);
    }

    if ((itp->it_nempty >= itp->it_hiwat) &&
        (icp->ic_state == AV1394_IC_DMA)) {
        av1394_ic_trigger_softintr(icp, icp->ic_num,
            AV1394_PREQ_IT_UNDERRUN);
    }
    mutex_exit(&icp->ic_mutex);
    mutex_exit(&ip->i_mutex);

    AV1394_TNF_EXIT(av1394_it_ixl_eof_cb);
}

void
av1394_it_underrun(av1394_ic_t *icp)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_inst_t       *avp = icp->ic_avp;
    int         idx;
    ixl1394_jump_t      *old_jmp;
    ixl1394_jump_t      new_jmp;
    id1394_isoch_dma_updateinfo_t update_info;
    int         err;
    int         result;

    AV1394_TNF_ENTER(av1394_it_underrun);

    /*
     * update the last full frame's jump to NULL
     */
    idx = (itp->it_first_empty + icp->ic_nframes - 1) % icp->ic_nframes;

    old_jmp = &itp->it_frame_info[idx].fi_last_buf->tb_jump;
    itp->it_saved_label = old_jmp->label;

    new_jmp.ixl_opcode = IXL1394_OP_JUMP_U;
    new_jmp.label = NULL;
    new_jmp.next_ixlp = NULL;

    update_info.orig_ixlp = (ixl1394_command_t *)old_jmp;
    update_info.temp_ixlp = (ixl1394_command_t *)&new_jmp;
    update_info.ixl_count = 1;

    mutex_exit(&icp->ic_mutex);
    err = t1394_update_isoch_dma(avp->av_t1394_hdl,
        icp->ic_isoch_hdl, &update_info, 0, &result);
    mutex_enter(&icp->ic_mutex);

    if (err == DDI_SUCCESS) {
        itp->it_underrun_idx = idx;
        icp->ic_state = AV1394_IC_SUSPENDED;
        cv_signal(&icp->ic_xfer_cv);
    } else {
        TNF_PROBE_2(av1394_it_underrun_error_update,
            AV1394_TNF_ISOCH_ERROR, "", tnf_int, err, err,
            tnf_int, result, result);
    }

    AV1394_TNF_EXIT(av1394_it_underrun);
}

/*
 * resume from the underrun condition
 */
static int
av1394_it_underrun_resume(av1394_ic_t *icp)
{
    av1394_it_t     *itp = &icp->ic_it;
    av1394_inst_t       *avp = icp->ic_avp;
    av1394_it_ixl_buf_t *bp;
    int         idx;
    ixl1394_jump_t      *old_jmp;
    ixl1394_jump_t      new_jmp;
    id1394_isoch_dma_updateinfo_t update_info;
    int         err;
    int         result;

    AV1394_TNF_ENTER(av1394_it_underrun_resume);

    /*
     * resuming the transfer it a lot like starting the transfer:
     * first the IXL begin block needs to be executed, then the rest
     * of the IXL chain. The following dynamic updates are needed:
     *
     *  1. update the begin block to jump to the first empty frame;
     *  2. restore the original jump label which we previously
     *    changed to jump to the underrun block;
     *
     * update #1
     *   start by updating the begin block with a new buffer address
     */
    idx = (itp->it_underrun_idx + 1) % icp->ic_nframes;
    bp = itp->it_frame_info[idx].fi_first_buf;
    av1394_it_ixl_begin_update_pkts(icp, bp);

    old_jmp = &itp->it_ixl_begin.be_jump;

    new_jmp.ixl_opcode = IXL1394_OP_JUMP_U;
    new_jmp.label = (ixl1394_command_t *)&bp->tb_label;
    new_jmp.next_ixlp = NULL;

    update_info.orig_ixlp = (ixl1394_command_t *)old_jmp;
    update_info.temp_ixlp = (ixl1394_command_t *)&new_jmp;
    update_info.ixl_count = 1;

    mutex_exit(&icp->ic_mutex);
    err = t1394_update_isoch_dma(avp->av_t1394_hdl,
        icp->ic_isoch_hdl, &update_info, 0, &result);
    mutex_enter(&icp->ic_mutex);

    if (err != DDI_SUCCESS) {
        TNF_PROBE_2(av1394_it_underrun_resume_error_update1,
            AV1394_TNF_ISOCH_ERROR, "", tnf_int, err, err,
            tnf_int, result, result);
        AV1394_TNF_EXIT(av1394_it_underrun_resume);
        return (EIO);
    }

    /*
     * update #2
     */
    bp = itp->it_frame_info[itp->it_underrun_idx].fi_last_buf;
    old_jmp = &bp->tb_jump;

    new_jmp.ixl_opcode = IXL1394_OP_JUMP_U;
    new_jmp.label = itp->it_saved_label;
    new_jmp.next_ixlp = NULL;

    update_info.orig_ixlp = (ixl1394_command_t *)old_jmp;
    update_info.temp_ixlp = (ixl1394_command_t *)&new_jmp;
    update_info.ixl_count = 1;

    mutex_exit(&icp->ic_mutex);
    err = t1394_update_isoch_dma(avp->av_t1394_hdl,
        icp->ic_isoch_hdl, &update_info, 0, &result);
    mutex_enter(&icp->ic_mutex);

    if (err != DDI_SUCCESS) {
        TNF_PROBE_2(av1394_it_underrun_resume_error_update2,
            AV1394_TNF_ISOCH_ERROR, "", tnf_int, err, err,
            tnf_int, result, result);
        AV1394_TNF_EXIT(av1394_it_underrun_resume);
        return (EIO);
    }

    icp->ic_state = AV1394_IC_DMA;

    AV1394_TNF_EXIT(av1394_it_underrun_resume);
    return (0);
}

/*ARGSUSED*/
static void
av1394_it_dma_stopped_cb(t1394_isoch_dma_handle_t t1394_idma_hdl,
    opaque_t idma_evt_arg, id1394_isoch_dma_stopped_t status)
{
    av1394_ic_t *icp = idma_evt_arg;

    AV1394_TNF_ENTER(av1394_it_dma_stopped_cb);

    mutex_enter(&icp->ic_mutex);
    icp->ic_state = AV1394_IC_IDLE;
    mutex_exit(&icp->ic_mutex);

    AV1394_TNF_EXIT(av1394_it_dma_stopped_cb);
}


/*
 *
 * --- data transfer routines
 *
 * av1394_it_add_frames()
 *    Add full frames to the pool.
 */
static int
av1394_it_add_frames(av1394_ic_t *icp, int idx, int cnt)
{
    av1394_it_t *itp = &icp->ic_it;
    av1394_it_frame_info_t *fip;
    int     prev_full = itp->it_last_full;
    uint16_t    syt;
    int     ret = 0;

    /* can only add to tail */
    if (idx != ((itp->it_last_full + 1) % icp->ic_nframes)) {
        TNF_PROBE_1(av1394_it_add_frames_error,
            AV1394_TNF_ISOCH_ERROR, "", tnf_int, idx, idx);
        return (EINVAL);
    }

    /* turn empty frames into full ones */
    itp->it_nempty -= cnt;
    itp->it_first_empty = (itp->it_first_empty + cnt) % icp->ic_nframes;
    itp->it_nfull += cnt;
    itp->it_last_full = (itp->it_last_full + cnt) % icp->ic_nframes;
    ASSERT((itp->it_nempty >= 0) && (itp->it_nfull <= icp->ic_nframes));

    /*
     * update frame timestamps if needed
     */
    if (AV1394_TS_MODE_GET_SIZE(icp->ic_param.cp_ts_mode) > 0) {
        ASSERT(prev_full >= 0);
        fip = &itp->it_frame_info[prev_full];
        syt = *(uint16_t *)fip->fi_ts_off;
        syt = av1394_it_ts_syt_inc(syt, fip->fi_ncycs);
        av1394_it_update_frame_syt(icp, idx, cnt, syt);
    }

    av1394_it_dma_sync_frames(icp, idx, cnt);

    /* if suspended due to overrun, check if we can resume */
    if ((icp->ic_state == AV1394_IC_SUSPENDED) &&
        (itp->it_nempty >= itp->it_lowat)) {
        ret = av1394_it_underrun_resume(icp);
    }

    return (ret);
}

/*
 * wait for empty frames
 */
static int
av1394_it_wait_frames(av1394_ic_t *icp, int *idx, int *cnt, int *nlost)
{
    av1394_it_t *itp = &icp->ic_it;
    int     ret = 0;

    while ((itp->it_nempty == 0) && (icp->ic_state == AV1394_IC_DMA)) {
        if (cv_wait_sig(&icp->ic_xfer_cv, &icp->ic_mutex) <= 0) {
            ret = EINTR;
            break;
        }
    }

    if (itp->it_nempty > 0) {
        *idx = itp->it_first_empty;
        *cnt = itp->it_nempty;
        *nlost = 0;
        ret = 0;
    }
    return (ret);
}

/*
 * update frame timestamps for a range of frames
 */
static void
av1394_it_update_frame_syt(av1394_ic_t *icp, int first, int cnt, uint16_t syt)
{
    av1394_it_t *itp = &icp->ic_it;
    int     i;
    int     j = first;  /* frame number */

    for (i = cnt; i > 0; i--) {
        *(uint16_t *)itp->it_frame_info[j].fi_ts_off = syt;
        syt = av1394_it_ts_syt_inc(syt, itp->it_frame_info[j].fi_ncycs);
        j = (j + 1) % icp->ic_nframes;
    }
}

/*
 * convert cycle timestamp into SYT timestamp:
 *
 * Cycle timer:          cycleSeconds         cycleCount     cycleOffset
 *                   31_30_29_28_27_26_25  24___15_14_13_12  11________0
 * Cycle timestamp:              |------------------------|
 * SYT timestamp:                               |----------------------|
 */
static uint16_t
av1394_it_ts_cyc2syt(uint16_t cyc)
{
    return (((cyc & 0xF) << 12) + 0x800);
}

/*
 * increment SYT by a number of cycles
 */
static uint16_t
av1394_it_ts_syt_inc(uint16_t syt, uint16_t ncycs)
{
    return (syt + (ncycs << 12));
}

/*
 * copyin from the kernel buffer
 */
static void
av1394_it_kcopyin(av1394_ic_t *icp, void *buf, size_t len)
{
    av1394_it_t *itp = &icp->ic_it;
    av1394_isoch_seg_t *seg = itp->it_data_pool.ip_seg;

    ASSERT(itp->it_write_off + len < icp->ic_framesz);

    bcopy(buf, seg[itp->it_write_idx].is_kaddr + itp->it_write_off, len);
    itp->it_write_off += len;
}

/*
 * copyin from the user buffer
 */
static int
av1394_it_copyin(av1394_ic_t *icp, struct uio *uiop, int *full_cnt, int dv)
{
    av1394_it_t *itp = &icp->ic_it;
    av1394_isoch_seg_t *seg = itp->it_data_pool.ip_seg;
    int     idx = itp->it_write_idx;
    int     framesz = icp->ic_framesz;
    size_t      len, frame_resid, start_resid;
    caddr_t     kaddr, kaddr_end;
    int     ret = 0;

    *full_cnt = 0;

    while ((uiop->uio_resid > 0) && (itp->it_write_cnt > 0)) {
        kaddr = seg[idx].is_kaddr + itp->it_write_off;
        frame_resid = framesz - itp->it_write_off;
        len = min(uiop->uio_resid, frame_resid);

        mutex_exit(&icp->ic_mutex);
        ret = uiomove(kaddr, len, UIO_WRITE, uiop);
        mutex_enter(&icp->ic_mutex);
        if (ret != 0) {
            break;
        }

        itp->it_write_off += len;
        if ((itp->it_write_off == framesz) && dv) {
            /*
             * for DV formats, make sure we got a frame start.
             * this is to ensure correct timestamping
             */
            kaddr = seg[idx].is_kaddr;
            kaddr_end = kaddr + framesz;
            while (!av1394_it_is_dv_frame_start(kaddr)) {
                kaddr += icp->ic_pktsz;
                if (kaddr == kaddr_end) {
                    break;
                }
            }
            start_resid = kaddr_end - kaddr;
            if (start_resid != framesz) {
                bcopy(kaddr, seg[idx].is_kaddr, start_resid);
                itp->it_write_off = start_resid;
            }
        }
        if (itp->it_write_off == framesz) {
            /* for DV formats, reset frame's SYT fields */
            if (dv) {
                av1394_it_reset_frame_syt(icp, idx);
            }
            itp->it_write_off = 0;
            itp->it_write_cnt--;
            idx = (idx + 1) % icp->ic_nframes;
            (*full_cnt)++;
        }
    }

    return (ret);
}

/*
 * check if a packet starts a new DV frame
 */
static boolean_t
av1394_it_is_dv_frame_start(caddr_t kaddr)
{
    uint8_t *p = (uint8_t *)kaddr + 8;
    /*
     * in the DIF block ID data, which immediately follows CIP header,
     * SCT, Dseq and DBN fields should be zero (Ref: IEC 61834-2, Fig. 66)
     */
    return (((p[0] & 0xE0) == 0) && ((p[1] & 0xF0) == 0) && (p[2] == 0));
}

/*
 * reset all frame's SYT fields
 */
static void
av1394_it_reset_frame_syt(av1394_ic_t *icp, int idx)
{
    caddr_t     kaddr = icp->ic_it.it_data_pool.ip_seg[idx].is_kaddr;
    caddr_t     kaddr_end = kaddr + icp->ic_framesz;

    kaddr += 6;
    while (kaddr < kaddr_end) {
        *(uint16_t *)kaddr = 0xFFFF;
        kaddr += icp->ic_pktsz;
    }
}