Lines Matching refs:sq
467 #define SQPUT_Q(sq, qp) \
469 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
474 EQUIV(sq->sq_head, sq->sq_tail); \
476 ASSERT(sq == qp->q_syncq); \
478 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
480 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
481 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
487 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
489 EQUIV(sq->sq_head, sq->sq_nqueues); \
490 if (sq->sq_head == NULL) { \
491 sq->sq_head = sq->sq_tail = qp; \
492 sq->sq_flags |= SQ_MESSAGES; \
494 qp->q_sqprev = sq->sq_tail; \
495 sq->sq_tail->q_sqnext = qp; \
496 sq->sq_tail = qp; \
502 queue_t **qpp = &sq->sq_tail; \
514 sq->sq_head = qp; \
515 sq->sq_pri = sq->sq_head->q_spri; \
521 sq->sq_nqueues++; \
529 #define SQRM_Q(sq, qp) \
531 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
533 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
534 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
536 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
537 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
538 ASSERT(sq->sq_nqueues != 0); \
541 sq->sq_head = qp->q_sqnext; \
548 sq->sq_tail = qp->q_sqprev; \
557 if (sq->sq_head != NULL) { \
558 sq->sq_pri = sq->sq_head->q_spri; \
560 sq->sq_flags &= ~SQ_MESSAGES; \
561 sq->sq_pri = 0; \
563 sq->sq_nqueues--; \
564 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
565 (sq->sq_flags & SQ_QUEUED) == 0); \
595 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \
596 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
597 if ((sq)->sq_ciputctrl != NULL) { \
599 int nlocks = (sq)->sq_nciputctrl; \
600 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
601 ASSERT((sq)->sq_type & SQ_CIPUT); \
610 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \
611 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
612 if ((sq)->sq_ciputctrl != NULL) { \
614 int nlocks = (sq)->sq_nciputctrl; \
615 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
616 ASSERT((sq)->sq_type & SQ_CIPUT); \
690 syncq_t *sq = &qip->qu_syncq;
722 sq->sq_head = NULL;
723 sq->sq_tail = NULL;
724 sq->sq_evhead = NULL;
725 sq->sq_evtail = NULL;
726 sq->sq_callbpend = NULL;
727 sq->sq_outer = NULL;
728 sq->sq_onext = NULL;
729 sq->sq_oprev = NULL;
730 sq->sq_next = NULL;
731 sq->sq_svcflags = 0;
732 sq->sq_servcount = 0;
733 sq->sq_needexcl = 0;
734 sq->sq_nqueues = 0;
735 sq->sq_pri = 0;
737 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
738 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
739 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
751 syncq_t *sq = &qip->qu_syncq;
766 mutex_destroy(&sq->sq_lock);
767 cv_destroy(&sq->sq_wait);
768 cv_destroy(&sq->sq_exitwait);
778 syncq_t *sq = buf;
782 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
783 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
784 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
793 syncq_t *sq = buf;
795 ASSERT(sq->sq_head == NULL);
796 ASSERT(sq->sq_tail == NULL);
797 ASSERT(sq->sq_evhead == NULL);
798 ASSERT(sq->sq_evtail == NULL);
799 ASSERT(sq->sq_callbpend == NULL);
800 ASSERT(sq->sq_callbflags == 0);
801 ASSERT(sq->sq_outer == NULL);
802 ASSERT(sq->sq_onext == NULL);
803 ASSERT(sq->sq_oprev == NULL);
804 ASSERT(sq->sq_next == NULL);
805 ASSERT(sq->sq_needexcl == 0);
806 ASSERT(sq->sq_svcflags == 0);
807 ASSERT(sq->sq_servcount == 0);
808 ASSERT(sq->sq_nqueues == 0);
809 ASSERT(sq->sq_pri == 0);
810 ASSERT(sq->sq_count == 0);
811 ASSERT(sq->sq_rmqcount == 0);
812 ASSERT(sq->sq_cancelid == 0);
813 ASSERT(sq->sq_ciputctrl == NULL);
814 ASSERT(sq->sq_nciputctrl == 0);
815 ASSERT(sq->sq_type == 0);
816 ASSERT(sq->sq_flags == 0);
818 mutex_destroy(&sq->sq_lock);
819 cv_destroy(&sq->sq_wait);
820 cv_destroy(&sq->sq_exitwait);
1682 * sq could be either outer or inner.
1686 wait_syncq(syncq_t *sq)
1690 mutex_enter(SQLOCK(sq));
1691 count = sq->sq_count;
1692 SQ_PUTLOCKS_ENTER(sq);
1693 SUM_SQ_PUTCOUNTS(sq, count);
1695 sq->sq_flags |= SQ_WANTWAKEUP;
1696 SQ_PUTLOCKS_EXIT(sq);
1697 cv_wait(&sq->sq_wait, SQLOCK(sq));
1698 count = sq->sq_count;
1699 SQ_PUTLOCKS_ENTER(sq);
1700 SUM_SQ_PUTCOUNTS(sq, count);
1702 SQ_PUTLOCKS_EXIT(sq);
1703 mutex_exit(SQLOCK(sq));
1713 syncq_t *sq = q->q_syncq;
1715 mutex_enter(SQLOCK(sq));
1717 sq->sq_flags |= SQ_WANTWAKEUP;
1718 cv_wait(&sq->sq_wait, SQLOCK(sq));
1720 mutex_exit(SQLOCK(sq));
2101 syncq_t *sq = wrq->q_syncq;
2103 mutex_enter(SQLOCK(sq));
2105 sq->sq_flags |= SQ_WANTWAKEUP;
2106 cv_wait(&sq->sq_wait, SQLOCK(sq));
2108 mutex_exit(SQLOCK(sq));
2113 syncq_t *sq, *outer;
2139 sq = rq->q_syncq;
2140 if ((outer = sq->sq_outer) != NULL) {
2167 wait_syncq(sq);
2179 syncq_t *sq = rq->q_syncq;
2181 mutex_enter(SQLOCK(sq));
2183 sq->sq_flags |= SQ_WANTWAKEUP;
2184 cv_wait(&sq->sq_wait, SQLOCK(sq));
2186 mutex_exit(SQLOCK(sq));
2490 syncq_t *sq, *outer;
2509 sq = rq->q_syncq;
2510 outer = sq->sq_outer;
2517 ASSERT(sq->sq_outer == NULL);
2518 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2520 if (sq != SQ(rq)) {
2522 free_syncq(sq);
2527 if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2559 sq = SQ(rq);
2560 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2561 ASSERT(sq->sq_outer == NULL);
2562 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2568 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2570 rq->q_syncq = wq->q_syncq = sq;
2571 sq->sq_type = sqtype;
2572 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2580 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2581 sq->sq_svcflags = 0;
2601 sq = new_syncq();
2602 sq->sq_type = rq->q_syncq->sq_type;
2603 sq->sq_flags = rq->q_syncq->sq_flags;
2604 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2605 sq->sq_oprev == NULL);
2606 wq->q_syncq = sq;
2609 sq = dmp->dm_sq;
2615 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2616 sq->sq_oprev == NULL);
2617 rq->q_syncq = wq->q_syncq = sq;
2645 syncq_t *sq;
2663 sq = new_syncq();
2665 sq->sq_type = sqtype | SQ_PERMOD;
2666 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2669 sq->sq_onext = sq->sq_oprev = sq;
2673 dmp->dm_sq = sq;
2683 free_syncq(sq);
3353 syncq_t *sq;
3359 sq = &qip->qu_syncq;
3367 qp->q_syncq = sq;
3383 wqp->q_syncq = sq;
3392 sq->sq_count = 0;
3393 sq->sq_rmqcount = 0;
3394 sq->sq_flags = 0;
3395 sq->sq_type = 0;
3396 sq->sq_callbflags = 0;
3397 sq->sq_cancelid = 0;
3398 sq->sq_ciputctrl = NULL;
3399 sq->sq_nciputctrl = 0;
3400 sq->sq_needexcl = 0;
3401 sq->sq_svcflags = 0;
3415 syncq_t *sq, *outer;
3465 sq = SQ(qp);
3466 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3467 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3468 ASSERT(sq->sq_outer == NULL);
3469 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3470 ASSERT(sq->sq_callbpend == NULL);
3471 ASSERT(sq->sq_needexcl == 0);
3473 if (sq->sq_ciputctrl != NULL) {
3474 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3475 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3476 sq->sq_nciputctrl, 0);
3478 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3479 sq->sq_ciputctrl = NULL;
3480 sq->sq_nciputctrl = 0;
3761 syncq_t *sq;
3782 DQ(sq, sqhead, sqtail, sq_next);
3783 ASSERT(sq != NULL);
3784 ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3786 syncq_service(sq);
3798 wait_sq_svc(syncq_t *sq)
3800 mutex_enter(SQLOCK(sq));
3801 sq->sq_svcflags |= SQ_DISABLED;
3802 if (sq->sq_svcflags & SQ_BGTHREAD) {
3807 ASSERT(sq->sq_servcount == 1);
3809 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3812 sq->sq_svcflags &= ~SQ_BGTHREAD;
3813 sq->sq_servcount = 0;
3818 while (sq->sq_servcount != 0) {
3819 sq->sq_flags |= SQ_WANTWAKEUP;
3820 cv_wait(&sq->sq_wait, SQLOCK(sq));
3823 mutex_exit(SQLOCK(sq));
3839 sqenable(syncq_t *sq)
3847 ASSERT(MUTEX_HELD(SQLOCK(sq)));
3849 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3850 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3856 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3860 * Check whether we should enable sq at all.
3867 if (sq->sq_servcount != 0) {
3868 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3869 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3875 sq->sq_tstamp = ddi_get_lbolt();
3879 sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3880 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3881 if (sq->sq_servid != NULL) {
3882 sq->sq_servcount++;
3891 if (sq->sq_servcount != 0)
3901 ENQUEUE(sq, sqhead, sqtail, sq_next);
3902 sq->sq_svcflags |= SQ_BGTHREAD;
3903 sq->sq_servcount = 1;
4622 syncq_t *sq = sql->sql_sq;
4625 mutex_enter(SQLOCK(sq));
4626 count = sq->sq_count;
4627 ASSERT(sq->sq_rmqcount <= count);
4628 SQ_PUTLOCKS_ENTER(sq);
4629 SUM_SQ_PUTCOUNTS(sq, count);
4630 if (count == sq->sq_rmqcount)
4658 sq->sq_needexcl++;
4659 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4660 while (count > sq->sq_rmqcount) {
4661 sq->sq_flags |= SQ_WANTWAKEUP;
4662 SQ_PUTLOCKS_EXIT(sq);
4663 cv_wait(&sq->sq_wait, SQLOCK(sq));
4664 count = sq->sq_count;
4665 SQ_PUTLOCKS_ENTER(sq);
4666 SUM_SQ_PUTCOUNTS(sq, count);
4668 sq->sq_needexcl--;
4669 if (sq->sq_needexcl == 0)
4670 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4671 SQ_PUTLOCKS_EXIT(sq);
4672 ASSERT(count == sq->sq_rmqcount);
4673 mutex_exit(SQLOCK(sq));
4859 syncq_t *sq = qp->q_syncq;
4874 mutex_enter(SQLOCK(sq));
4875 if (sq->sq_flags & SQ_WANTWAKEUP) {
4876 sq->sq_flags &= ~SQ_WANTWAKEUP;
4877 cv_broadcast(&sq->sq_wait);
4879 mutex_exit(SQLOCK(sq));
4881 if (wsq != sq) {
4905 mutex_enter(SQLOCK(sq));
4906 sq->sq_rmqcount++;
4907 if (sq->sq_flags & SQ_WANTWAKEUP) {
4908 sq->sq_flags &= ~SQ_WANTWAKEUP;
4909 cv_broadcast(&sq->sq_wait);
4911 mutex_exit(SQLOCK(sq));
4991 mutex_enter(SQLOCK(sq));
4992 sq->sq_rmqcount--;
4993 mutex_exit(SQLOCK(sq));
5009 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5013 mutex_enter(SQLOCK(sq));
5022 count = sq->sq_count;
5023 SQ_PUTLOCKS_ENTER(sq);
5024 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5025 SUM_SQ_PUTCOUNTS(sq, count);
5027 sq->sq_needexcl++;
5028 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5030 while ((sq->sq_flags & flag) ||
5032 sq->sq_flags |= SQ_WANTWAKEUP;
5034 SQ_PUTLOCKS_EXIT(sq);
5036 cv_wait(&sq->sq_wait, SQLOCK(sq));
5038 count = sq->sq_count;
5039 SQ_PUTLOCKS_ENTER(sq);
5040 SUM_SQ_PUTCOUNTS(sq, count);
5043 sq->sq_needexcl--;
5044 sq->sq_flags |= flag;
5047 if (sq->sq_needexcl == 0) {
5048 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5050 SQ_PUTLOCKS_EXIT(sq);
5051 } else if (sq->sq_needexcl == 0) {
5052 SQ_PUTCOUNT_SETFAST(sq);
5055 mutex_exit(SQLOCK(sq));
5071 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5075 mutex_enter(SQLOCK(sq));
5077 ASSERT(sq->sq_flags & resetflag);
5078 flags = sq->sq_flags & ~resetflag;
5079 sq->sq_flags = flags;
5083 cv_broadcast(&sq->sq_wait);
5085 sq->sq_flags = flags;
5089 drain_syncq(sq);
5094 mutex_exit(SQLOCK(sq));
5106 dropsq(syncq_t *sq, uint16_t resetflag)
5110 mutex_enter(SQLOCK(sq));
5111 ASSERT(sq->sq_flags & resetflag);
5112 flags = sq->sq_flags & ~resetflag;
5115 cv_broadcast(&sq->sq_wait);
5117 sq->sq_flags = flags;
5118 mutex_exit(SQLOCK(sq));
5131 emptysq(syncq_t *sq)
5135 mutex_enter(SQLOCK(sq));
5136 flags = sq->sq_flags;
5142 if (sq->sq_count == 0) {
5144 drain_syncq(sq);
5147 sqenable(sq);
5149 mutex_exit(SQLOCK(sq));
5380 entersq(syncq_t *sq, int entrypoint)
5392 ASSERT(sq);
5394 mutex_enter(SQLOCK(sq));
5395 flags = sq->sq_flags;
5396 type = sq->sq_type;
5399 count = sq->sq_count;
5400 SQ_PUTLOCKS_ENTER(sq);
5401 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5402 SUM_SQ_PUTCOUNTS(sq, count);
5403 sq->sq_needexcl++;
5404 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5414 sq->sq_flags = flags | SQ_WANTWAKEUP;
5416 SQ_PUTLOCKS_EXIT(sq);
5418 cv_wait(&sq->sq_wait, SQLOCK(sq));
5420 count = sq->sq_count;
5421 SQ_PUTLOCKS_ENTER(sq);
5422 SUM_SQ_PUTCOUNTS(sq, count);
5424 flags = sq->sq_flags;
5428 ASSERT(sq->sq_needexcl > 0);
5429 sq->sq_needexcl--;
5430 if (sq->sq_needexcl == 0) {
5431 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5445 SQ_PUTLOCKS_EXIT(sq);
5447 mutex_exit(SQLOCK(sq));
5448 outer_enter(sq->sq_outer, SQ_GOAWAY);
5449 mutex_enter(SQLOCK(sq));
5450 flags = sq->sq_flags;
5456 count = sq->sq_count;
5462 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5465 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5469 sq->sq_flags = flags | SQ_WANTWAKEUP;
5470 cv_wait(&sq->sq_wait, SQLOCK(sq));
5471 count = sq->sq_count;
5472 flags = sq->sq_flags;
5476 sq->sq_count++;
5477 ASSERT(sq->sq_count != 0); /* Wraparound */
5480 ASSERT(sq->sq_count == 1);
5481 sq->sq_flags |= SQ_EXCL;
5483 SQ_PUTLOCKS_EXIT(sq);
5486 mutex_exit(SQLOCK(sq));
5499 leavesq(syncq_t *sq, int entrypoint)
5512 ASSERT(sq);
5514 mutex_enter(SQLOCK(sq));
5515 flags = sq->sq_flags;
5516 type = sq->sq_type;
5521 cv_broadcast(&sq->sq_wait);
5525 cv_broadcast(&sq->sq_exitwait);
5533 ASSERT(sq->sq_count != 0);
5534 sq->sq_count--;
5536 sq->sq_flags = flags & ~SQ_EXCL;
5537 drain_syncq(sq);
5538 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5542 outer_exit(sq->sq_outer);
5546 ASSERT(sq->sq_count != 0);
5547 sq->sq_count--;
5549 sq->sq_flags = flags & ~SQ_EXCL;
5550 mutex_exit(SQLOCK(sq));
5553 if (!(sq->sq_type & c_outer))
5554 outer_exit(sq->sq_outer);
5566 syncq_t *sq = qp->q_syncq;
5568 mutex_enter(SQLOCK(sq));
5569 sq->sq_count++;
5570 ASSERT(sq->sq_count != 0); /* Wraparound */
5571 mutex_exit(SQLOCK(sq));
5583 syncq_t *sq = qp->q_syncq;
5586 mutex_enter(SQLOCK(sq));
5587 ASSERT(sq->sq_count > 0);
5588 sq->sq_count--;
5590 flags = sq->sq_flags;
5594 cv_broadcast(&sq->sq_wait);
5596 sq->sq_flags = flags;
5603 if (sq->sq_count == 0) {
5604 drain_syncq(sq);
5607 sqenable(sq);
5610 mutex_exit(SQLOCK(sq));
5649 free_syncq(syncq_t *sq)
5651 ASSERT(sq->sq_head == NULL);
5652 ASSERT(sq->sq_outer == NULL);
5653 ASSERT(sq->sq_callbpend == NULL);
5654 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5655 (sq->sq_onext == sq && sq->sq_oprev == sq));
5657 if (sq->sq_ciputctrl != NULL) {
5658 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5659 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5660 sq->sq_nciputctrl, 0);
5662 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5665 sq->sq_tail = NULL;
5666 sq->sq_evhead = NULL;
5667 sq->sq_evtail = NULL;
5668 sq->sq_ciputctrl = NULL;
5669 sq->sq_nciputctrl = 0;
5670 sq->sq_count = 0;
5671 sq->sq_rmqcount = 0;
5672 sq->sq_callbflags = 0;
5673 sq->sq_cancelid = 0;
5674 sq->sq_next = NULL;
5675 sq->sq_needexcl = 0;
5676 sq->sq_svcflags = 0;
5677 sq->sq_nqueues = 0;
5678 sq->sq_pri = 0;
5679 sq->sq_onext = NULL;
5680 sq->sq_oprev = NULL;
5681 sq->sq_flags = 0;
5682 sq->sq_type = 0;
5683 sq->sq_servcount = 0;
5685 kmem_cache_free(syncq_cache, sq);
5727 syncq_t *sq;
5752 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5753 mutex_enter(SQLOCK(sq));
5754 count = sq->sq_count;
5755 SQ_PUTLOCKS_ENTER(sq);
5756 sq->sq_flags |= SQ_WRITER;
5757 SUM_SQ_PUTCOUNTS(sq, count);
5760 SQ_PUTLOCKS_EXIT(sq);
5761 mutex_exit(SQLOCK(sq));
5777 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5778 mutex_enter(SQLOCK(sq));
5779 count = sq->sq_count;
5780 SQ_PUTLOCKS_ENTER(sq);
5781 SUM_SQ_PUTCOUNTS(sq, count);
5783 sq->sq_flags |= SQ_WANTWAKEUP;
5784 SQ_PUTLOCKS_EXIT(sq);
5785 cv_wait(&sq->sq_wait, SQLOCK(sq));
5786 count = sq->sq_count;
5787 SQ_PUTLOCKS_ENTER(sq);
5788 SUM_SQ_PUTCOUNTS(sq, count);
5790 SQ_PUTLOCKS_EXIT(sq);
5791 mutex_exit(SQLOCK(sq));
5817 syncq_t *sq;
5848 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5849 drain_needed += dropsq(sq, SQ_WRITER);
5855 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5856 emptysq(sq);
5879 outer_insert(syncq_t *outer, syncq_t *sq)
5883 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5884 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */
5891 mutex_enter(SQLOCK(sq));
5892 sq->sq_outer = outer;
5893 outer->sq_onext->sq_oprev = sq;
5894 sq->sq_onext = outer->sq_onext;
5895 outer->sq_onext = sq;
5896 sq->sq_oprev = outer;
5897 mutex_exit(SQLOCK(sq));
5910 outer_remove(syncq_t *outer, syncq_t *sq)
5914 ASSERT(sq->sq_outer == outer);
5921 mutex_enter(SQLOCK(sq));
5922 sq->sq_outer = NULL;
5923 sq->sq_onext->sq_oprev = sq->sq_oprev;
5924 sq->sq_oprev->sq_onext = sq->sq_onext;
5925 sq->sq_oprev = sq->sq_onext = NULL;
5926 mutex_exit(SQLOCK(sq));
5979 syncq_t *osq, *sq, *outer;
6046 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6048 uint_t maxcnt = (sq == osq) ? 1 : 0;
6050 mutex_enter(SQLOCK(sq));
6051 count = sq->sq_count;
6052 SQ_PUTLOCKS_ENTER(sq);
6053 SUM_SQ_PUTCOUNTS(sq, count);
6054 if (sq->sq_count > maxcnt)
6056 sq->sq_flags |= SQ_WRITER;
6057 SQ_PUTLOCKS_EXIT(sq);
6058 mutex_exit(SQLOCK(sq));
6172 sq_run_events(syncq_t *sq)
6176 uint16_t flags = sq->sq_flags;
6179 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6180 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6181 sq->sq_oprev == NULL) ||
6182 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6183 sq->sq_oprev != NULL));
6186 ASSERT(sq->sq_count == 1);
6199 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6200 ASSERT(bp->b_queue->q_syncq == sq);
6201 ASSERT(sq->sq_flags & SQ_EVENTS);
6211 ASSERT(sq->sq_evhead == bp);
6212 sq->sq_evhead = bp->b_next;
6216 ASSERT(sq->sq_evtail == bp);
6217 sq->sq_evtail = NULL;
6218 sq->sq_flags &= ~SQ_EVENTS;
6223 mutex_exit(SQLOCK(sq));
6227 mutex_enter(SQLOCK(sq));
6231 flags = sq->sq_flags;
6234 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6235 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6239 cv_broadcast(&sq->sq_wait);
6243 cv_broadcast(&sq->sq_exitwait);
6245 sq->sq_flags = flags;
6256 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6260 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6270 if (sq->sq_evhead == NULL) {
6271 sq->sq_evhead = sq->sq_evtail = mp;
6272 sq->sq_flags |= SQ_EVENTS;
6274 ASSERT(sq->sq_evtail != NULL);
6275 ASSERT(sq->sq_evtail->b_next == NULL);
6276 ASSERT(sq->sq_flags & SQ_EVENTS);
6277 sq->sq_evtail->b_next = mp;
6278 sq->sq_evtail = mp;
6291 count = sq->sq_count;
6292 SQ_PUTLOCKS_ENTER(sq);
6293 SUM_SQ_PUTCOUNTS(sq, count);
6307 SQ_PUTLOCKS_EXIT(sq);
6308 mutex_exit(SQLOCK(sq));
6314 ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6315 sq->sq_flags |= SQ_EXCL;
6316 ASSERT(sq->sq_count == 0);
6317 sq->sq_count++;
6318 SQ_PUTLOCKS_EXIT(sq);
6321 sq_run_events(sq);
6326 sq->sq_count--;
6333 sq->sq_flags &= ~SQ_EXCL;
6338 ASSERT(!(sq->sq_flags & SQ_EXCL));
6347 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6348 drain_syncq(sq);
6350 mutex_exit(SQLOCK(sq));
6370 drain_syncq(syncq_t *sq)
6374 uint16_t type = sq->sq_type;
6375 uint16_t flags = sq->sq_flags;
6376 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE;
6379 "drain_syncq start:%p", sq);
6380 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6381 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6382 sq->sq_oprev == NULL) ||
6383 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6384 sq->sq_oprev != NULL));
6390 sq->sq_svcflags &= ~SQ_SERVICE;
6398 ASSERT(sq->sq_servcount != 0);
6399 sq->sq_servcount--;
6401 mutex_exit(SQLOCK(sq));
6414 cv_broadcast(&sq->sq_wait);
6418 cv_broadcast(&sq->sq_exitwait);
6420 sq->sq_flags = flags;
6422 ASSERT(sq->sq_servcount != 0);
6423 sq->sq_servcount--;
6425 mutex_exit(SQLOCK(sq));
6436 type = sq->sq_type;
6438 if (sq->sq_count > 1) {
6440 ASSERT(sq->sq_servcount != 0);
6441 sq->sq_servcount--;
6443 mutex_exit(SQLOCK(sq));
6446 sq->sq_flags |= SQ_EXCL;
6460 sq->sq_count++;
6461 ASSERT(sq->sq_count != 0); /* wraparound */
6463 while ((flags = sq->sq_flags) & SQ_QUEUED) {
6477 if (sq->sq_evhead != NULL) {
6478 ASSERT(sq->sq_flags & SQ_EVENTS);
6480 count = sq->sq_count;
6481 SQ_PUTLOCKS_ENTER(sq);
6482 SUM_SQ_PUTCOUNTS(sq, count);
6484 SQ_PUTLOCKS_EXIT(sq);
6489 sq->sq_flags = flags | SQ_EXCL;
6490 SQ_PUTLOCKS_EXIT(sq);
6495 sq_run_events(sq);
6503 ASSERT(sq->sq_flags & SQ_EXCL);
6504 sq->sq_flags &= ~SQ_EXCL;
6511 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6515 ASSERT(sq->sq_evhead == NULL);
6516 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6529 for (qp = sq->sq_head;
6549 mutex_exit(SQLOCK(sq));
6551 qdrain_syncq(sq, qp);
6552 mutex_enter(SQLOCK(sq));
6563 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6564 flags = sq->sq_flags;
6567 * sq->sq_head cannot change because we hold the
6575 * possibility is to yank the queue out of the sq list and put it onto
6579 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6580 (type & SQ_CI) || sq->sq_head->q_draining);
6590 cv_broadcast(&sq->sq_wait);
6594 cv_broadcast(&sq->sq_exitwait);
6596 sq->sq_flags = flags;
6598 ASSERT(sq->sq_count != 0);
6600 sq->sq_count--;
6603 ASSERT(sq->sq_servcount != 0);
6604 sq->sq_servcount--;
6607 mutex_exit(SQLOCK(sq));
6610 "drain_syncq end:%p", sq);
6645 qdrain_syncq(syncq_t *sq, queue_t *q)
6653 "drain_syncq start:%p", sq);
6654 ASSERT(q->q_syncq == sq);
6656 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6662 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6663 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6664 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6668 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6669 sq->sq_oprev == NULL) ||
6670 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6671 sq->sq_oprev != NULL));
6673 count = sq->sq_count;
6681 SUM_SQ_PUTCOUNTS(sq, count);
6727 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6739 ASSERT(bp->b_queue->q_syncq == sq);
6794 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6795 mutex_enter(SQLOCK(sq));
6796 sq->sq_flags &= ~SQ_EXCL;
6797 mutex_exit(SQLOCK(sq));
6807 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6810 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6815 mutex_enter(SQLOCK(sq));
6817 SQRM_Q(sq, q);
6818 mutex_exit(SQLOCK(sq));
6839 "drain_syncq end:%p", sq);
6854 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6855 * SQLOCK(sq) and QLOCK(q) are not held.
6858 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6860 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6862 ASSERT(sq->sq_count > 0);
6863 ASSERT(q->q_syncq == sq);
6864 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6865 sq->sq_oprev == NULL) ||
6866 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6867 sq->sq_oprev != NULL));
6885 ASSERT(q->q_syncq == sq);
6892 mutex_enter(SQLOCK(sq));
6902 SQPUT_Q(sq, q);
6909 ASSERT(sq->sq_tail != NULL);
6910 if (sq->sq_tail == sq->sq_head) {
6917 ASSERT(sq->sq_flags & SQ_QUEUED);
6926 sq->sq_count--;
6928 putnext_tail(sq, q, 0);
6929 /* Should not reference sq or q after this point. */
6951 flush_syncq(syncq_t *sq, queue_t *qp)
6957 mutex_enter(SQLOCK(sq));
6964 if (qp != NULL && sq->sq_evhead != NULL) {
6965 ASSERT(sq->sq_flags & SQ_EVENTS);
6968 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6978 if (bp == sq->sq_evtail) {
6980 sq->sq_evtail = mp_prev;
6983 sq->sq_evhead = mp_next;
6984 if (sq->sq_evhead == NULL)
6985 sq->sq_flags &= ~SQ_EVENTS;
7000 q = sq->sq_head;
7002 ASSERT(q->q_syncq == sq);
7016 mp_head->b_queue->q_syncq == sq);
7031 SQRM_Q(sq, q);
7037 * going to drop SQLOCK(sq) and return. We wakeup syncq
7040 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7041 sq->sq_flags &= ~SQ_WANTWAKEUP;
7042 cv_broadcast(&sq->sq_wait);
7045 mutex_exit(SQLOCK(sq));
7057 mutex_enter(SQLOCK(sq));
7062 q = sq->sq_head;
7067 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7070 if (sq->sq_flags & SQ_WANTWAKEUP) {
7071 sq->sq_flags &= ~SQ_WANTWAKEUP;
7072 cv_broadcast(&sq->sq_wait);
7075 mutex_exit(SQLOCK(sq));
7095 syncq_t *sq;
7106 sq = qp->q_syncq;
7107 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7109 SQ_PUTLOCKS_HELD(sq);
7115 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7136 SQRM_Q(sq, qp);
7200 if (sq->sq_evhead != NULL) {
7201 ASSERT(sq->sq_flags & SQ_EVENTS);
7203 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7213 if (bp == sq->sq_evtail) {
7215 sq->sq_evtail = prev;
7218 sq->sq_evhead = next;
7219 if (sq->sq_evhead == NULL)
7220 sq->sq_flags &= ~SQ_EVENTS;
7229 flags = sq->sq_flags;
7234 cv_broadcast(&sq->sq_wait);
7236 sq->sq_flags = flags;
7254 syncq_t *sq = q->q_syncq;
7257 mutex_enter(SQLOCK(sq));
7258 count = sq->sq_count;
7259 SQ_PUTLOCKS_ENTER(sq);
7260 SUM_SQ_PUTCOUNTS(sq, count);
7262 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7270 sq->sq_flags |= SQ_EXCL;
7271 SQ_PUTLOCKS_EXIT(sq);
7272 mutex_exit(SQLOCK(sq));
7284 SQ_PUTLOCKS_EXIT(sq);
7285 sqfill_events(sq, q, mp, func);
7295 * Acquires SQLOCK(sq) if non-NULL is returned.
7298 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7319 cbp->cbp_sq = sq;
7322 mutex_enter(SQLOCK(sq));
7323 cbp->cbp_next = sq->sq_callbpend;
7324 sq->sq_callbpend = cbp;
7329 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7333 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7335 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7347 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7351 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7353 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7367 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7374 syncq_t *sq;
7379 sq = cbp->cbp_sq;
7380 mutex_enter(SQLOCK(sq));
7381 type = sq->sq_type;
7383 count = sq->sq_count;
7384 SQ_PUTLOCKS_ENTER(sq);
7385 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7386 SUM_SQ_PUTCOUNTS(sq, count);
7387 sq->sq_needexcl++;
7388 ASSERT(sq->sq_needexcl != 0); /* wraparound */
7394 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7395 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7396 (sq->sq_cancelid == cbp->cbp_id)) {
7398 sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7399 callbparams_free(sq, cbp);
7401 ASSERT(sq->sq_needexcl > 0);
7402 sq->sq_needexcl--;
7403 if (sq->sq_needexcl == 0) {
7404 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7406 SQ_PUTLOCKS_EXIT(sq);
7408 mutex_exit(SQLOCK(sq));
7411 sq->sq_flags |= SQ_WANTWAKEUP;
7413 SQ_PUTLOCKS_EXIT(sq);
7415 cv_wait(&sq->sq_wait, SQLOCK(sq));
7417 count = sq->sq_count;
7418 SQ_PUTLOCKS_ENTER(sq);
7419 SUM_SQ_PUTCOUNTS(sq, count);
7423 sq->sq_count++;
7424 ASSERT(sq->sq_count != 0); /* Wraparound */
7427 sq->sq_flags |= SQ_EXCL;
7428 ASSERT(sq->sq_needexcl > 0);
7429 sq->sq_needexcl--;
7430 if (sq->sq_needexcl == 0) {
7431 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7433 SQ_PUTLOCKS_EXIT(sq);
7436 mutex_exit(SQLOCK(sq));
7444 mutex_enter(SQLOCK(sq));
7445 callbparams_free(sq, cbp);
7446 mutex_exit(SQLOCK(sq));
7447 leavesq(sq, SQ_CALLBACK);
7469 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7471 uint16_t flags = sq->sq_flags;
7473 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7482 cv_broadcast(&sq->sq_wait);
7486 cv_broadcast(&sq->sq_exitwait);
7488 sq->sq_flags = flags;
7498 (sq->sq_svcflags & SQ_DISABLED)) {
7500 drain_syncq(sq);
7503 sqenable(sq);
7507 mutex_exit(SQLOCK(sq));
7509 "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7812 syncq_t *sq;
7814 sq = _RD(q)->q_syncq;
7815 if (sq->sq_flags & SQ_BLOCKED)
7816 unblocksq(sq, SQ_BLOCKED, 0);
8281 syncq_service(syncq_t *sq)
8284 mutex_enter(SQLOCK(sq));
8285 ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8286 ASSERT(sq->sq_servcount != 0);
8287 ASSERT(sq->sq_next == NULL);
8290 if (sq->sq_svcflags & SQ_BGTHREAD)
8291 sq->sq_svcflags &= ~SQ_BGTHREAD;
8294 sq->sq_svcflags |= SQ_SERVICE;
8295 drain_syncq(sq);