Lines Matching defs:rwl
26 #define VALID_RWLOCK(rwl) ISC_MAGIC_VALID(rwl, RWLOCK_MAGIC)
44 isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type);
52 print_lock(const char *operation, isc_rwlock_t *rwl, isc_rwlocktype_t type) {
61 rwl, isc_thread_self(), operation,
67 rwl->write_requests, rwl->write_completions,
68 rwl->cnt_and_flag, rwl->readers_waiting,
69 rwl->write_granted, rwl->write_quota);
76 rwl, isc_thread_self(), operation,
82 (rwl->type == isc_rwlocktype_read ?
87 rwl->active, rwl->granted,
88 rwl->readers_waiting, rwl->writers_waiting);
94 isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota,
99 REQUIRE(rwl != NULL);
105 rwl->magic = 0;
107 rwl->spins = 0;
109 rwl->write_requests = 0;
110 rwl->write_completions = 0;
111 rwl->cnt_and_flag = 0;
112 rwl->readers_waiting = 0;
113 rwl->write_granted = 0;
120 rwl->write_quota = write_quota;
122 rwl->type = isc_rwlocktype_read;
123 rwl->original = isc_rwlocktype_none;
124 rwl->active = 0;
125 rwl->granted = 0;
126 rwl->readers_waiting = 0;
127 rwl->writers_waiting = 0;
130 rwl->read_quota = read_quota;
133 rwl->write_quota = write_quota;
136 result = isc_mutex_init(&rwl->lock);
140 result = isc_condition_init(&rwl->readable);
150 result = isc_condition_init(&rwl->writeable);
161 rwl->magic = RWLOCK_MAGIC;
166 (void)isc_condition_destroy(&rwl->readable);
168 DESTROYLOCK(&rwl->lock);
174 isc_rwlock_destroy(isc_rwlock_t *rwl) {
175 REQUIRE(VALID_RWLOCK(rwl));
178 REQUIRE(rwl->write_requests == rwl->write_completions &&
179 rwl->cnt_and_flag == 0 && rwl->readers_waiting == 0);
181 LOCK(&rwl->lock);
182 REQUIRE(rwl->active == 0 &&
183 rwl->readers_waiting == 0 &&
184 rwl->writers_waiting == 0);
185 UNLOCK(&rwl->lock);
188 rwl->magic = 0;
189 (void)isc_condition_destroy(&rwl->readable);
190 (void)isc_condition_destroy(&rwl->writeable);
191 DESTROYLOCK(&rwl->lock);
262 isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
265 REQUIRE(VALID_RWLOCK(rwl));
269 ISC_MSG_PRELOCK, "prelock"), rwl, type);
273 if (rwl->write_requests != rwl->write_completions) {
275 LOCK(&rwl->lock);
276 if (rwl->write_requests != rwl->write_completions) {
277 rwl->readers_waiting++;
278 WAIT(&rwl->readable, &rwl->lock);
279 rwl->readers_waiting--;
281 UNLOCK(&rwl->lock);
285 cntflag = atomic_fetch_add_explicit(&rwl->cnt_and_flag,
289 cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR);
293 if ((rwl->cnt_and_flag & WRITER_ACTIVE) == 0)
297 LOCK(&rwl->lock);
298 rwl->readers_waiting++;
299 if ((rwl->cnt_and_flag & WRITER_ACTIVE) != 0)
300 WAIT(&rwl->readable, &rwl->lock);
301 rwl->readers_waiting--;
302 UNLOCK(&rwl->lock);
335 rwl->write_granted = 0;
341 prev_writer = atomic_fetch_add_explicit(&rwl->write_requests, 1,
344 prev_writer = isc_atomic_xadd(&rwl->write_requests, 1);
346 while (rwl->write_completions != prev_writer) {
347 LOCK(&rwl->lock);
348 if (rwl->write_completions != prev_writer) {
349 WAIT(&rwl->writeable, &rwl->lock);
350 UNLOCK(&rwl->lock);
353 UNLOCK(&rwl->lock);
361 (&rwl->cnt_and_flag, &cntflag2, WRITER_ACTIVE,
365 cntflag2 = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0,
373 LOCK(&rwl->lock);
374 if (rwl->cnt_and_flag != 0)
375 WAIT(&rwl->writeable, &rwl->lock);
376 UNLOCK(&rwl->lock);
379 INSIST((rwl->cnt_and_flag & WRITER_ACTIVE) != 0);
380 rwl->write_granted++;
385 ISC_MSG_POSTLOCK, "postlock"), rwl, type);
392 isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
394 isc_int32_t max_cnt = rwl->spins * 2 + 10;
402 result = isc__rwlock_lock(rwl, type);
408 } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS);
410 rwl->spins += (cnt - rwl->spins) / 8;
416 isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
419 REQUIRE(VALID_RWLOCK(rwl));
423 ISC_MSG_PRELOCK, "prelock"), rwl, type);
428 if (rwl->write_requests != rwl->write_completions)
433 cntflag = atomic_fetch_add_explicit(&rwl->cnt_and_flag,
437 cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR);
446 (&rwl->cnt_and_flag, READER_INCR,
449 cntflag = isc_atomic_xadd(&rwl->cnt_and_flag,
457 rwl->write_completions != rwl->write_requests) {
458 LOCK(&rwl->lock);
459 BROADCAST(&rwl->writeable);
460 UNLOCK(&rwl->lock);
470 (&rwl->cnt_and_flag, &zero, WRITER_ACTIVE,
474 cntflag = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0,
485 atomic_fetch_sub_explicit(&rwl->write_completions, 1,
488 (void)isc_atomic_xadd(&rwl->write_completions, -1);
491 rwl->write_granted++;
496 ISC_MSG_POSTLOCK, "postlock"), rwl, type);
503 isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
504 REQUIRE(VALID_RWLOCK(rwl));
512 (&rwl->cnt_and_flag, &reader_incr, WRITER_ACTIVE,
526 atomic_fetch_sub_explicit(&rwl->write_completions, 1,
537 prevcnt = isc_atomic_cmpxchg(&rwl->cnt_and_flag,
551 (void)isc_atomic_xadd(&rwl->write_completions, -1);
561 isc_rwlock_downgrade(isc_rwlock_t *rwl) {
564 REQUIRE(VALID_RWLOCK(rwl));
569 prev_readers = atomic_fetch_add_explicit(&rwl->cnt_and_flag,
576 atomic_fetch_sub_explicit(&rwl->cnt_and_flag, WRITER_ACTIVE,
578 atomic_fetch_add_explicit(&rwl->write_completions, 1,
584 prev_readers = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR);
589 (void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE);
590 (void)isc_atomic_xadd(&rwl->write_completions, 1);
595 LOCK(&rwl->lock);
596 if (rwl->readers_waiting > 0)
597 BROADCAST(&rwl->readable);
598 UNLOCK(&rwl->lock);
602 isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
605 REQUIRE(VALID_RWLOCK(rwl));
609 ISC_MSG_PREUNLOCK, "preunlock"), rwl, type);
614 prev_cnt = atomic_fetch_sub_explicit(&rwl->cnt_and_flag,
618 prev_cnt = isc_atomic_xadd(&rwl->cnt_and_flag, -READER_INCR);
626 rwl->write_completions != rwl->write_requests) {
627 LOCK(&rwl->lock);
628 BROADCAST(&rwl->writeable);
629 UNLOCK(&rwl->lock);
639 atomic_fetch_sub_explicit(&rwl->cnt_and_flag, WRITER_ACTIVE,
641 atomic_fetch_add_explicit(&rwl->write_completions, 1,
644 (void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE);
645 (void)isc_atomic_xadd(&rwl->write_completions, 1);
648 if (rwl->write_granted >= rwl->write_quota ||
649 rwl->write_requests == rwl->write_completions ||
650 (rwl->cnt_and_flag & ~WRITER_ACTIVE) != 0) {
659 LOCK(&rwl->lock);
660 if (rwl->readers_waiting > 0) {
662 BROADCAST(&rwl->readable);
664 UNLOCK(&rwl->lock);
667 if (rwl->write_requests != rwl->write_completions &&
669 LOCK(&rwl->lock);
670 BROADCAST(&rwl->writeable);
671 UNLOCK(&rwl->lock);
678 rwl, type);
687 doit(isc_rwlock_t *rwl, isc_rwlocktype_t type, isc_boolean_t nonblock) {
692 REQUIRE(VALID_RWLOCK(rwl));
694 LOCK(&rwl->lock);
698 ISC_MSG_PRELOCK, "prelock"), rwl, type);
702 if (rwl->readers_waiting != 0)
706 ((rwl->active == 0 ||
707 (rwl->type == isc_rwlocktype_read &&
708 (rwl->writers_waiting == 0 ||
709 rwl->granted < rwl->read_quota)))))
711 rwl->type = isc_rwlocktype_read;
712 rwl->active++;
713 rwl->granted++;
720 rwl->readers_waiting++;
721 WAIT(&rwl->readable, &rwl->lock);
722 rwl->readers_waiting--;
726 if (rwl->writers_waiting != 0)
729 if (!skip && rwl->active == 0) {
730 rwl->type = isc_rwlocktype_write;
731 rwl->active = 1;
732 rwl->granted++;
739 rwl->writers_waiting++;
740 WAIT(&rwl->writeable, &rwl->lock);
741 rwl->writers_waiting--;
748 ISC_MSG_POSTLOCK, "postlock"), rwl, type);
751 UNLOCK(&rwl->lock);
757 isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
759 isc_int32_t max_cnt = rwl->spins * 2 + 10;
767 result = doit(rwl, type, ISC_FALSE);
773 } while (doit(rwl, type, ISC_TRUE) != ISC_R_SUCCESS);
775 rwl->spins += (cnt - rwl->spins) / 8;
781 isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
782 return (doit(rwl, type, ISC_TRUE));
786 isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
789 REQUIRE(VALID_RWLOCK(rwl));
790 LOCK(&rwl->lock);
791 REQUIRE(rwl->type == isc_rwlocktype_read);
792 REQUIRE(rwl->active != 0);
795 if (rwl->active == 1) {
796 rwl->original = (rwl->original == isc_rwlocktype_none) ?
798 rwl->type = isc_rwlocktype_write;
802 UNLOCK(&rwl->lock);
807 isc_rwlock_downgrade(isc_rwlock_t *rwl) {
809 REQUIRE(VALID_RWLOCK(rwl));
810 LOCK(&rwl->lock);
811 REQUIRE(rwl->type == isc_rwlocktype_write);
812 REQUIRE(rwl->active == 1);
814 rwl->type = isc_rwlocktype_read;
815 rwl->original = (rwl->original == isc_rwlocktype_none) ?
821 if (rwl->original == isc_rwlocktype_none &&
822 (rwl->writers_waiting == 0 || rwl->granted < rwl->read_quota) &&
823 rwl->readers_waiting > 0)
824 BROADCAST(&rwl->readable);
826 UNLOCK(&rwl->lock);
830 isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
832 REQUIRE(VALID_RWLOCK(rwl));
833 LOCK(&rwl->lock);
834 REQUIRE(rwl->type == type);
840 ISC_MSG_PREUNLOCK, "preunlock"), rwl, type);
843 INSIST(rwl->active > 0);
844 rwl->active--;
845 if (rwl->active == 0) {
846 if (rwl->original != isc_rwlocktype_none) {
847 rwl->type = rwl->original;
848 rwl->original = isc_rwlocktype_none;
850 if (rwl->type == isc_rwlocktype_read) {
851 rwl->granted = 0;
852 if (rwl->writers_waiting > 0) {
853 rwl->type = isc_rwlocktype_write;
854 SIGNAL(&rwl->writeable);
855 } else if (rwl->readers_waiting > 0) {
857 BROADCAST(&rwl->readable);
860 if (rwl->readers_waiting > 0) {
861 if (rwl->writers_waiting > 0 &&
862 rwl->granted < rwl->write_quota) {
863 SIGNAL(&rwl->writeable);
865 rwl->granted = 0;
866 rwl->type = isc_rwlocktype_read;
867 BROADCAST(&rwl->readable);
869 } else if (rwl->writers_waiting > 0) {
870 rwl->granted = 0;
871 SIGNAL(&rwl->writeable);
873 rwl->granted = 0;
877 INSIST(rwl->original == isc_rwlocktype_none);
882 rwl, type);
885 UNLOCK(&rwl->lock);
894 isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota,
897 REQUIRE(rwl != NULL);
902 rwl->type = isc_rwlocktype_read;
903 rwl->active = 0;
904 rwl->magic = RWLOCK_MAGIC;
910 isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
911 REQUIRE(VALID_RWLOCK(rwl));
914 if (rwl->type != isc_rwlocktype_read && rwl->active != 0)
916 rwl->type = isc_rwlocktype_read;
917 rwl->active++;
919 if (rwl->active != 0)
921 rwl->type = isc_rwlocktype_write;
922 rwl->active = 1;
928 isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
929 return (isc_rwlock_lock(rwl, type));
933 isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
936 REQUIRE(VALID_RWLOCK(rwl));
937 REQUIRE(rwl->type == isc_rwlocktype_read);
938 REQUIRE(rwl->active != 0);
941 if (rwl->active == 1)
942 rwl->type = isc_rwlocktype_write;
949 isc_rwlock_downgrade(isc_rwlock_t *rwl) {
951 REQUIRE(VALID_RWLOCK(rwl));
952 REQUIRE(rwl->type == isc_rwlocktype_write);
953 REQUIRE(rwl->active == 1);
955 rwl->type = isc_rwlocktype_read;
959 isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
960 REQUIRE(VALID_RWLOCK(rwl));
961 REQUIRE(rwl->type == type);
965 INSIST(rwl->active > 0);
966 rwl->active--;
972 isc_rwlock_destroy(isc_rwlock_t *rwl) {
973 REQUIRE(rwl != NULL);
974 REQUIRE(rwl->active == 0);
975 rwl->magic = 0;