Lines Matching defs:__
55 #define __ _masm->
60 #define BLOCK_COMMENT(str) __ block_comment(str)
96 __ inc_counter(&counter, t1, t2);
104 address start = __ pc();
145 __ ld_ptr(thread.as_address(), G2_thread);
146 __ reinit_heapbase();
152 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), t);
153 __ br_null_short(t, Assembler::pt, L);
154 __ stop("StubRoutines::call_stub: entered with pending exception");
155 __ bind(L);
161 __ ld_ptr(parameter_size.as_address(), t); // get parameter size (in words)
162 __ add(t, frame::memory_parameter_word_sp_offset, t); // add space for save area (in words)
163 __ round_to(t, WordsPerLong); // make sure it is multiple of 2 (in words)
164 __ sll(t, Interpreter::logStackElementSize, t); // compute number of bytes
165 __ neg(t); // negate so it can be used with save
166 __ save(SP, t, SP); // setup new frame
207 __ ld_ptr(parameter_size.as_in().as_address(), cnt); // parameter counter
208 __ add( FP, STACK_BIAS, dst );
209 __ cmp_zero_and_br(Assembler::zero, cnt, exit);
210 __ delayed()->sub(dst, BytesPerWord, dst); // setup Lentry_args
214 __ BIND(loop);
216 __ ld_ptr(src, 0, tmp);
217 __ add(src, BytesPerWord, src);
218 __ st_ptr(tmp, dst, 0);
219 __ deccc(cnt);
220 __ br(Assembler::greater, false, Assembler::pt, loop);
221 __ delayed()->sub(dst, Interpreter::stackElementSize, dst);
224 __ BIND(exit);
232 __ mov(SP, saved_SP); // keep track of SP before call
237 __ ld_ptr(parameter_size.as_in().as_address(), t); // get parameter size (in words)
238 __ sll(t, Interpreter::logStackElementSize, t); // compute number of bytes
239 __ sub(FP, t, Gargs); // setup parameter pointer
241 __ add( Gargs, STACK_BIAS, Gargs ); // Account for LP64 stack bias
243 __ mov(SP, O5_savedSP);
254 __ jmpl(entry_point.as_in().as_register(), G0, O7);
255 __ delayed()->mov(method.as_in().as_register(), G5_method); // setup method
258 return_pc = __ pc();
269 __ cmp(type, T_OBJECT); __ br(Assembler::equal, false, Assembler::pn, is_object);
270 __ delayed()->cmp(type, T_FLOAT); __ br(Assembler::equal, false, Assembler::pn, is_float);
271 __ delayed()->cmp(type, T_DOUBLE); __ br(Assembler::equal, false, Assembler::pn, is_double);
272 __ delayed()->cmp(type, T_LONG); __ br(Assembler::equal, false, Assembler::pn, is_long);
273 __ delayed()->nop();
276 __ st(O0, addr, G0);
278 __ BIND(exit);
279 __ ret();
280 __ delayed()->restore();
282 __ BIND(is_object);
283 __ ba(exit);
284 __ delayed()->st_ptr(O0, addr, G0);
286 __ BIND(is_float);
287 __ ba(exit);
288 __ delayed()->stf(FloatRegisterImpl::S, F0, addr, G0);
290 __ BIND(is_double);
291 __ ba(exit);
292 __ delayed()->stf(FloatRegisterImpl::D, F0, addr, G0);
294 __ BIND(is_long);
296 __ ba(exit);
297 __ delayed()->st_long(O0, addr, G0); // store entire long
308 __ ba(exit);
309 __ delayed()->stx(G1, addr, G0); // store entire long
311 __ st(O1, addr, BytesPerInt);
312 __ ba(exit);
313 __ delayed()->st(O0, addr, G0);
331 address start = __ pc();
333 __ verify_thread();
341 __ verify_oop(Oexception);
342 __ st_ptr(Oexception, pending_exception_addr);
343 __ set((intptr_t)__FILE__, temp_reg);
344 __ st_ptr(temp_reg, exception_file_offset_addr);
345 __ set((intptr_t)__LINE__, temp_reg);
346 __ st(temp_reg, exception_line_offset_addr);
352 __ jump_to(stub_ret, temp_reg);
353 __ delayed()->nop();
369 address start = __ pc();
382 __ ld_ptr(exception_addr, Gtemp);
383 __ br_notnull_short(Gtemp, Assembler::pt, L);
384 __ stop("StubRoutines::forward exception: no pending exception (1)");
385 __ bind(L);
390 __ get_thread();
391 __ ld_ptr(exception_addr, Oexception);
392 __ verify_oop(Oexception);
393 __ save_frame(0); // compensates for compiler weakness
394 __ add(O7->after_save(), frame::pc_return_offset, Lscratch); // save the issuing PC
396 __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), G2_thread, Lscratch);
397 __ mov(O0, handler_reg);
398 __ restore(); // compensates for compiler weakness
400 __ ld_ptr(exception_addr, Oexception);
401 __ add(O7, frame::pc_return_offset, Oissuing_pc); // save the issuing PC
406 __ br_notnull_short(Oexception, Assembler::pt, L);
407 __ stop("StubRoutines::forward exception: no pending exception (2)");
408 __ bind(L);
412 __ jmp(handler_reg, 0);
414 __ delayed()->st_ptr(G0, exception_addr);
434 // properly traversed and ignored during GC, so we change the meaning of the "__"
436 #undef __
437 #define __ masm->
451 __ verify_thread();
455 __ assert_not_delayed();
461 __ save_frame(0);
463 int frame_complete = __ offset();
468 __ set_last_Java_frame(last_java_sp, G0);
469 if (VerifyThread) __ mov(G2_thread, O0); // about to be smashed; pass early
470 __ save_thread(noreg);
473 __ mov(arg1, O1);
476 __ mov(arg2, O2);
480 __ call(runtime_entry, relocInfo::runtime_call_type);
482 __ delayed()->mov(G2_thread, O0); // pass thread as first argument
484 __ delayed()->nop(); // (thread already passed)
485 __ restore_thread(noreg);
486 __ reset_last_Java_frame();
494 __ ld_ptr(exception_addr, scratch_reg);
495 __ br_notnull_short(scratch_reg, Assembler::pt, L);
496 __ should_not_reach_here();
497 __ bind(L);
500 __ call(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
502 __ delayed()->restore();
508 #undef __
509 #define __ _masm->
516 address start = __ pc();
520 __ save_frame(0);
525 __ set((intptr_t)&zero, L0); __ ldf( FloatRegisterImpl::S, L0, 0, F0);
526 __ set((intptr_t)&one, L0); __ ldf( FloatRegisterImpl::S, L0, 0, F1); // 1.0 to F1
530 __ fadd( FloatRegisterImpl::S, F1, as_FloatRegister(i-1), as_FloatRegister(i));
534 __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, F2, F16 );
535 __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, F18, F18 );
539 __ fadd( FloatRegisterImpl::D, F16, as_FloatRegister(i-2), as_FloatRegister(i));
545 __ set( i, as_iRegister(i));
546 __ set(16 + i, as_oRegister(i));
547 __ set(24 + i, as_gRegister(i));
549 __ set( 8 + i, as_lRegister(i));
552 __ stop("testing stop");
555 __ ret();
556 __ delayed()->restore();
564 address start = __ pc();
566 __ stop_subroutine();
573 address start = __ pc();
575 __ flush_windows();
576 __ retl(false);
577 __ delayed()->add( FP, STACK_BIAS, O0 );
589 __ set((intptr_t)lock_ptr, lock_ptr_reg);
593 __ set((intptr_t)lock_ptr, lock_ptr_reg);
594 __ and3(mark_oop_reg, StubRoutines::Sparc::v8_oop_lock_mask_in_place, scratch_reg);
595 __ add(lock_ptr_reg, scratch_reg, lock_ptr_reg);
602 __ set(StubRoutines::Sparc::locked, lock_reg);
604 __ mov(G0,yield_reg);
606 __ BIND(retry);
607 __ cmp_and_br_short(yield_reg, V8AtomicOperationUnderLockSpinCount, Assembler::less, Assembler::pt, dontyield);
615 __ save(SP, -96, SP);
616 __ save_all_globals_into_locals();
618 __ call(CAST_FROM_FN_PTR(address, os::naked_sleep));
619 __ delayed()->nop();
620 __ restore_globals_from_locals();
621 __ restore();
623 __ mov(G0,yield_reg);
625 __ BIND(dontyield);
628 __ swap(lock_ptr_reg, 0, lock_reg);
631 __ cmp(lock_reg, StubRoutines::Sparc::unlocked);
632 __ br(Assembler::notEqual, true, Assembler::pn, retry);
633 __ delayed()->add(yield_reg,1,yield_reg);
639 __ st(lock_reg, lock_ptr_reg, 0); // unlock
655 address start = __ pc();
661 __ BIND(retry);
662 __ mov(O0, O3); // scratch copy of exchange value
663 __ ld(O1, 0, O2); // observe the previous value
665 __ cas_under_lock(O1, O2, O3,
667 __ cmp_and_br_short(O2, O3, Assembler::notEqual, Assembler::pn, retry);
669 __ retl(false);
670 __ delayed()->mov(O2, O0); // report previous value to caller
674 __ retl(false);
675 __ delayed()->swap(O1, 0, O0);
686 __ swap(O1, 0, O0);
689 __ retl(false);
690 __ delayed()->nop();
714 address start = __ pc();
717 __ cas_under_lock(O1, O2, O0,
719 __ retl(false);
720 __ delayed()->nop();
744 address start = __ pc();
748 __ sllx(O0, 32, O0);
749 __ srl(O1, 0, O1);
750 __ or3(O0,O1,O0); // O0 holds 64-bit value from compare_value
751 __ sllx(O3, 32, O3);
752 __ srl(O4, 0, O4);
753 __ or3(O3,O4,O3); // O3 holds 64-bit value from exchange_value
754 __ casx(O2, O3, O0);
755 __ srl(O0, 0, O1); // unpacked return value in O1:O0
756 __ retl(false);
757 __ delayed()->srlx(O0, 32, O0);
779 address start = __ pc();
780 __ BIND(_atomic_add_stub);
784 __ BIND(retry);
786 __ lduw(O1, 0, O2);
787 __ add(O0, O2, O3);
788 __ cas(O1, O2, O3);
789 __ cmp_and_br_short(O2, O3, Assembler::notEqual, Assembler::pn, retry);
790 __ retl(false);
791 __ delayed()->add(O0, O2, O0); // note that cas made O2==O3
803 __ ld(O1, 0, value_reg);
804 __ add(O0, value_reg, value_reg);
805 __ st(value_reg, O1, 0);
808 __ membar(Assembler::StoreStore);
812 __ retl(false);
813 __ delayed()->mov(value_reg, O0);
836 address start = __ pc();
844 __ save_frame(0);
845 __ mov(G1, L1);
846 __ mov(G2, L2);
847 __ mov(G3, L3);
848 __ mov(G4, L4);
849 __ mov(G5, L5);
851 __ stf(FloatRegisterImpl::D, as_FloatRegister(i), preserve_addr, i * wordSize);
856 __ call(entry_point, relocInfo::runtime_call_type);
857 __ delayed()->nop();
859 __ mov(L1, G1);
860 __ mov(L2, G2);
861 __ mov(L3, G3);
862 __ mov(L4, G4);
863 __ mov(L5, G5);
865 __ ldf(FloatRegisterImpl::D, preserve_addr, as_FloatRegister(i), i * wordSize);
868 __ verify_thread();
870 __ jmp(O0, 0);
871 __ delayed()->restore();
886 __ align(CodeEntryAlignment);
888 address start = __ pc();
893 __ add(SP,-4*wordSize,SP); // Make space for 4 temps (stack must be 2 words aligned)
894 __ st_ptr(L0,SP,(frame::register_save_words+0)*wordSize);
895 __ st_ptr(L1,SP,(frame::register_save_words+1)*wordSize);
896 __ st_ptr(L2,SP,(frame::register_save_words+2)*wordSize);
897 __ st_ptr(L3,SP,(frame::register_save_words+3)*wordSize);
902 __ save_frame(0);
913 __ check_klass_subtype_slow_path(Rsub, Rsuper,
918 __ addcc(G0,0,Rret); // set Z flags, Z result
921 __ ld_ptr(SP,(frame::register_save_words+0)*wordSize,L0);
922 __ ld_ptr(SP,(frame::register_save_words+1)*wordSize,L1);
923 __ ld_ptr(SP,(frame::register_save_words+2)*wordSize,L2);
924 __ ld_ptr(SP,(frame::register_save_words+3)*wordSize,L3);
925 __ retl(); // Result in Rret is zero; flags set to Z
926 __ delayed()->add(SP,4*wordSize,SP);
928 __ ret(); // Result in Rret is zero; flags set to Z
929 __ delayed()->restore();
932 __ BIND(miss);
933 __ addcc(G0,1,Rret); // set NZ flags, NZ result
936 __ ld_ptr(SP,(frame::register_save_words+0)*wordSize,L0);
937 __ ld_ptr(SP,(frame::register_save_words+1)*wordSize,L1);
938 __ ld_ptr(SP,(frame::register_save_words+2)*wordSize,L2);
939 __ ld_ptr(SP,(frame::register_save_words+3)*wordSize,L3);
940 __ retl(); // Result in Rret is != 0; flags set to NZ
941 __ delayed()->add(SP,4*wordSize,SP);
943 __ ret(); // Result in Rret is != 0; flags set to NZ
944 __ delayed()->restore();
956 address start = __ pc();
958 __ verify_oop_subroutine();
974 __ signx(Rint, Rtmp);
975 __ cmp(Rint, Rtmp);
976 __ breakpoint_trap(Assembler::notEqual, Assembler::xcc);
1004 __ subcc(to, from, to_from);
1005 __ sll_ptr(count, log2_elem_size, byte_count);
1007 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pt, no_overlap_target);
1009 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pt, (*NOLp));
1010 __ delayed()->cmp(to_from, byte_count);
1012 __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, no_overlap_target);
1014 __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, (*NOLp));
1015 __ delayed()->nop();
1035 __ save_frame(0);
1038 __ mov(addr, L0);
1041 __ mov(count, L1);
1043 __ mov(addr->after_save(), O0);
1045 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre));
1046 __ delayed()->mov(count->after_save(), O1);
1048 __ mov(L0, addr);
1051 __ mov(L1, count);
1053 __ restore();
1083 __ save_frame(0);
1084 __ mov(addr->after_save(), O0);
1085 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post));
1086 __ delayed()->mov(count->after_save(), O1);
1087 __ restore();
1099 __ sll_ptr(count, LogBytesPerHeapOop, count);
1100 __ sub(count, BytesPerHeapOop, count);
1101 __ add(count, addr, count);
1103 __ srl_ptr(addr, CardTableModRefBS::card_shift, addr);
1104 __ srl_ptr(count, CardTableModRefBS::card_shift, count);
1105 __ sub(count, addr, count);
1107 __ set(rs, tmp);
1108 __ BIND(L_loop);
1109 __ stb(G0, tmp, addr);
1110 __ subcc(count, 1, count);
1111 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1112 __ delayed()->add(addr, 1, addr);
1147 __ set(block_copy_count, O4);
1148 __ cmp_and_br_short(count, O4, Assembler::lessUnsigned, Assembler::pt, L_skip_block_copy);
1152 __ sub(from, to, O4);
1153 __ srax(O4, 4, O4); // divide by 16 since following short branch have only 5 bits for imm.
1154 __ cmp_and_br_short(O4, (tail_size>>4), Assembler::lessEqualUnsigned, Assembler::pn, L_skip_block_copy);
1156 __ wrasi(G0, Assembler::ASI_ST_BLKINIT_PRIMARY);
1159 __ sub(count, (tail_size>>log2_elem_size), count); // count is still positive >= 0
1164 __ set(prefetch_count, O4);
1165 __ cmp_and_brx_short(count, O4, Assembler::less, Assembler::pt, L_block_copy);
1166 __ sub(count, prefetch_count, count);
1169 __ add(count, prefetch_count, count); // restore count
1174 __ add(count, (tail_size>>log2_elem_size), count); // restore count
1176 __ wrasi(G0, Assembler::ASI_PRIMARY_NOFAULT);
1178 __ membar(Assembler::StoreLoad);
1180 __ ba_short(L_copy);
1182 __ BIND(L_skip_block_copy);
1188 __ set(prefetch_count, O4);
1189 __ cmp_and_brx_short(count, O4, Assembler::lessUnsigned, Assembler::pt, L_copy);
1190 __ sub(count, prefetch_count, count);
1194 __ add(count, prefetch_count, count); // restore count
1212 __ align(OptoLoopAlignment);
1213 __ BIND(L_loop);
1216 __ prefetch(from, ArraycopySrcPrefetchDistance, Assembler::severalReads);
1219 __ prefetch(to, ArraycopyDstPrefetchDistance, Assembler::severalWritesAndPossiblyReads);
1222 __ ldx(from, 0, O4);
1223 __ ldx(from, 8, G4);
1224 __ inc(to, 16);
1225 __ inc(from, 16);
1226 __ deccc(count, count_dec); // Can we do next iteration after this one?
1227 __ srlx(O4, right_shift, G3);
1228 __ bset(G3, O3);
1229 __ sllx(O4, left_shift, O4);
1230 __ srlx(G4, right_shift, G3);
1231 __ bset(G3, O4);
1233 __ stxa(O3, to, -16);
1234 __ stxa(O4, to, -8);
1236 __ stx(O3, to, -16);
1237 __ stx(O4, to, -8);
1239 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1240 __ delayed()->sllx(G4, left_shift, O3);
1259 __ andcc(from, 7, G1); // misaligned bytes
1260 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1261 __ delayed()->nop();
1266 __ sll(G1, LogBitsPerByte, left_shift);
1267 __ mov(64, right_shift);
1268 __ sub(right_shift, left_shift, right_shift);
1274 __ dec(count, count_dec); // Pre-decrement 'count'
1275 __ andn(from, 7, from); // Align address
1276 __ ldx(from, 0, O3);
1277 __ inc(from, 8);
1278 __ sllx(O3, left_shift, O3);
1282 __ inccc(count, count_dec>>1 ); // + 8 bytes
1283 __ brx(Assembler::negative, true, Assembler::pn, L_copy_last_bytes);
1284 __ delayed()->inc(count, count_dec>>1); // restore 'count'
1287 __ ldx(from, 0, O4);
1288 __ inc(to, 8);
1289 __ inc(from, 8);
1290 __ srlx(O4, right_shift, G3);
1291 __ bset(O3, G3);
1292 __ stx(G3, to, -8);
1294 __ BIND(L_copy_last_bytes);
1295 __ srl(right_shift, LogBitsPerByte, right_shift); // misaligned bytes
1296 __ br(Assembler::always, false, Assembler::pt, L_copy_bytes);
1297 __ delayed()->sub(from, right_shift, from); // restore address
1299 __ BIND(L_aligned_copy);
1318 __ andcc(end_from, 7, G1); // misaligned bytes
1319 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1320 __ delayed()->deccc(count, count_dec); // Pre-decrement 'count'
1325 __ sll(G1, LogBitsPerByte, left_shift);
1326 __ mov(64, right_shift);
1327 __ sub(right_shift, left_shift, right_shift);
1333 __ andn(end_from, 7, end_from); // Align address
1334 __ ldx(end_from, 0, O3);
1335 __ align(OptoLoopAlignment);
1336 __ BIND(L_loop);
1337 __ ldx(end_from, -8, O4);
1338 __ deccc(count, count_dec); // Can we do next iteration after this one?
1339 __ ldx(end_from, -16, G4);
1340 __ dec(end_to, 16);
1341 __ dec(end_from, 16);
1342 __ srlx(O3, right_shift, O3);
1343 __ sllx(O4, left_shift, G3);
1344 __ bset(G3, O3);
1345 __ stx(O3, end_to, 8);
1346 __ srlx(O4, right_shift, O4);
1347 __ sllx(G4, left_shift, G3);
1348 __ bset(G3, O4);
1349 __ stx(O4, end_to, 0);
1350 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1351 __ delayed()->mov(G4, O3);
1353 __ inccc(count, count_dec>>1 ); // + 8 bytes
1354 __ brx(Assembler::negative, true, Assembler::pn, L_copy_last_bytes);
1355 __ delayed()->inc(count, count_dec>>1); // restore 'count'
1358 __ ldx(end_from, -8, O4);
1359 __ dec(end_to, 8);
1360 __ dec(end_from, 8);
1361 __ srlx(O3, right_shift, O3);
1362 __ sllx(O4, left_shift, G3);
1363 __ bset(O3, G3);
1364 __ stx(G3, end_to, 0);
1366 __ BIND(L_copy_last_bytes);
1367 __ srl(left_shift, LogBitsPerByte, left_shift); // misaligned bytes
1368 __ br(Assembler::always, false, Assembler::pt, L_copy_bytes);
1369 __ delayed()->add(end_from, left_shift, end_from); // restore address
1382 __ align(CodeEntryAlignment);
1384 address start = __ pc();
1398 *entry = __ pc();
1404 __ cmp(count, 23); // 16 + 7
1405 __ brx(Assembler::less, false, Assembler::pn, L_copy_byte);
1406 __ delayed()->mov(G0, offset);
1418 __ andcc(to, 7, G0);
1419 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment);
1420 __ delayed()->ld(from, 0, O3);
1421 __ inc(from, 4);
1422 __ inc(to, 4);
1423 __ dec(count, 4);
1424 __ st(O3, to, -4);
1425 __ BIND(L_skip_alignment);
1429 __ andcc(to, 7, G1); // misaligned bytes
1430 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1431 __ delayed()->neg(G1);
1432 __ inc(G1, 8); // bytes need to copy to next 8-bytes alignment
1433 __ sub(count, G1, count);
1434 __ BIND(L_align);
1435 __ ldub(from, 0, O3);
1436 __ deccc(G1);
1437 __ inc(from);
1438 __ stb(O3, to, 0);
1439 __ br(Assembler::notZero, false, Assembler::pt, L_align);
1440 __ delayed()->inc(to);
1441 __ BIND(L_skip_alignment);
1457 __ and3(count, 7, G4); // Save count
1458 __ srl(count, 3, count);
1460 __ mov(G4, count); // Restore count
1463 __ BIND(L_copy_byte);
1464 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1465 __ align(OptoLoopAlignment);
1466 __ BIND(L_copy_byte_loop);
1467 __ ldub(from, offset, O3);
1468 __ deccc(count);
1469 __ stb(O3, to, offset);
1470 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_byte_loop);
1471 __ delayed()->inc(offset);
1473 __ BIND(L_exit);
1476 __ retl();
1477 __ delayed()->mov(G0, O0); // return 0
1494 __ align(CodeEntryAlignment);
1496 address start = __ pc();
1510 *entry = __ pc();
1517 __ add(to, count, end_to); // offset after last copied element
1520 __ cmp(count, 23); // 16 + 7
1521 __ brx(Assembler::less, false, Assembler::pn, L_copy_byte);
1522 __ delayed()->add(from, count, end_from);
1529 __ andcc(end_to, 7, G1); // misaligned bytes
1530 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1531 __ delayed()->nop();
1532 __ sub(count, G1, count);
1533 __ BIND(L_align);
1534 __ dec(end_from);
1535 __ dec(end_to);
1536 __ ldub(end_from, 0, O3);
1537 __ deccc(G1);
1538 __ brx(Assembler::notZero, false, Assembler::pt, L_align);
1539 __ delayed()->stb(O3, end_to, 0);
1540 __ BIND(L_skip_alignment);
1547 __ dec(count, 16);
1561 __ align(OptoLoopAlignment);
1562 __ BIND(L_aligned_copy);
1563 __ dec(end_from, 16);
1564 __ ldx(end_from, 8, O3);
1565 __ ldx(end_from, 0, O4);
1566 __ dec(end_to, 16);
1567 __ deccc(count, 16);
1568 __ stx(O3, end_to, 8);
1569 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
1570 __ delayed()->stx(O4, end_to, 0);
1571 __ inc(count, 16);
1574 __ BIND(L_copy_byte);
1575 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1576 __ align(OptoLoopAlignment);
1577 __ BIND(L_copy_byte_loop);
1578 __ dec(end_from);
1579 __ dec(end_to);
1580 __ ldub(end_from, 0, O4);
1581 __ deccc(count);
1582 __ brx(Assembler::greater, false, Assembler::pt, L_copy_byte_loop);
1583 __ delayed()->stb(O4, end_to, 0);
1585 __ BIND(L_exit);
1588 __ retl();
1589 __ delayed()->mov(G0, O0); // return 0
1603 __ align(CodeEntryAlignment);
1605 address start = __ pc();
1619 *entry = __ pc();
1625 __ cmp(count, 11); // 8 + 3 (22 bytes)
1626 __ brx(Assembler::less, false, Assembler::pn, L_copy_2_bytes);
1627 __ delayed()->mov(G0, offset);
1639 __ andcc(to, 7, G0);
1640 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1641 __ delayed()->ld(from, 0, O3);
1642 __ inc(from, 4);
1643 __ inc(to, 4);
1644 __ dec(count, 2);
1645 __ st(O3, to, -4);
1646 __ BIND(L_skip_alignment);
1650 __ andcc(to, 3, G0);
1651 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1652 __ delayed()->lduh(from, 0, O3);
1653 __ inc(from, 2);
1654 __ inc(to, 2);
1655 __ dec(count);
1656 __ sth(O3, to, -2);
1657 __ BIND(L_skip_alignment);
1660 __ andcc(to, 7, G0);
1661 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment2);
1662 __ delayed()->lduh(from, 0, O3);
1663 __ dec(count, 2);
1664 __ lduh(from, 2, O4);
1665 __ inc(from, 4);
1666 __ inc(to, 4);
1667 __ sth(O3, to, -4);
1668 __ sth(O4, to, -2);
1669 __ BIND(L_skip_alignment2);
1685 __ and3(count, 3, G4); // Save
1686 __ srl(count, 2, count);
1688 __ mov(G4, count); // restore
1691 __ BIND(L_copy_2_bytes);
1692 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1693 __ align(OptoLoopAlignment);
1694 __ BIND(L_copy_2_bytes_loop);
1695 __ lduh(from, offset, O3);
1696 __ deccc(count);
1697 __ sth(O3, to, offset);
1698 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_2_bytes_loop);
1699 __ delayed()->inc(offset, 2);
1701 __ BIND(L_exit);
1704 __ retl();
1705 __ delayed()->mov(G0, O0); // return 0
1719 __ align(CodeEntryAlignment);
1721 address start = __ pc();
1751 __ and3(value, 0xff, value);
1752 __ sllx(value, 8, O3);
1753 __ or3(value, O3, value);
1757 __ sllx(value, 48, value);
1758 __ srlx(value, 48, value);
1761 __ sllx(value, 16, O3);
1762 __ or3(value, O3, value);
1765 __ cmp(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
1766 __ brx(Assembler::lessUnsigned, false, Assembler::pn, L_fill_elements); // use unsigned cmp
1767 __ delayed()->andcc(count, 1, G0);
1773 __ andcc(to, 1, G0);
1774 __ br(Assembler::zero, false, Assembler::pt, L_skip_align1);
1775 __ delayed()->nop();
1776 __ stb(value, to, 0);
1777 __ inc(to, 1);
1778 __ dec(count, 1);
1779 __ BIND(L_skip_align1);
1782 __ andcc(to, 2, G0);
1783 __ br(Assembler::zero, false, Assembler::pt, L_skip_align2);
1784 __ delayed()->nop();
1785 __ sth(value, to, 0);
1786 __ inc(to, 2);
1787 __ dec(count, 1 << (shift - 1));
1788 __ BIND(L_skip_align2);
1794 __ andcc(to, 7, G0);
1795 __ br(Assembler::zero, false, Assembler::pt, L_fill_32_bytes);
1796 __ delayed()->nop();
1797 __ stw(value, to, 0);
1798 __ inc(to, 4);
1799 __ dec(count, 1 << shift);
1800 __ BIND(L_fill_32_bytes);
1807 __ srl(value, 0, value);
1810 __ sllx(value, 32, O3);
1811 __ or3(value, O3, value);
1816 __ subcc(count, 8 << shift, count);
1817 __ brx(Assembler::less, false, Assembler::pt, L_check_fill_8_bytes);
1818 __ delayed()->nop();
1821 __ align(16);
1822 __ BIND(L_fill_32_bytes_loop);
1824 __ stx(value, to, 0);
1825 __ stx(value, to, 8);
1826 __ stx(value, to, 16);
1827 __ stx(value, to, 24);
1829 __ subcc(count, 8 << shift, count);
1830 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_fill_32_bytes_loop);
1831 __ delayed()->add(to, 32, to);
1833 __ BIND(L_check_fill_8_bytes);
1834 __ addcc(count, 8 << shift, count);
1835 __ brx(Assembler::zero, false, Assembler::pn, L_exit);
1836 __ delayed()->subcc(count, 1 << (shift + 1), count);
1837 __ brx(Assembler::less, false, Assembler::pn, L_fill_4_bytes);
1838 __ delayed()->andcc(count, 1<<shift, G0);
1844 __ BIND(L_fill_8_bytes_loop);
1845 __ stx(value, to, 0);
1846 __ subcc(count, 1 << (shift + 1), count);
1847 __ brx(Assembler::greaterEqual, false, Assembler::pn, L_fill_8_bytes_loop);
1848 __ delayed()->add(to, 8, to);
1851 __ andcc(count, 1<<shift, G0); // in delay slot of branches
1853 __ BIND(L_fill_elements);
1855 __ BIND(L_fill_4_bytes);
1856 __ brx(Assembler::zero, false, Assembler::pt, L_fill_2_bytes);
1858 __ delayed()->andcc(count, 1<<(shift-1), G0);
1860 __ delayed()->nop();
1862 __ stw(value, to, 0);
1864 __ inc(to, 4);
1866 __ andcc(count, 1<<(shift-1), G0); // in delay slot of branches
1867 __ BIND(L_fill_2_bytes);
1868 __ brx(Assembler::zero, false, Assembler::pt, L_fill_byte);
1869 __ delayed()->andcc(count, 1, count);
1870 __ sth(value, to, 0);
1872 __ inc(to, 2);
1874 __ andcc(count, 1, count); // in delay slot of branches
1875 __ BIND(L_fill_byte);
1876 __ brx(Assembler::zero, false, Assembler::pt, L_exit);
1877 __ delayed()->nop();
1878 __ stb(value, to, 0);
1880 __ BIND(L_fill_byte);
1883 __ BIND(L_fill_2_bytes);
1885 __ BIND(L_exit);
1886 __ retl();
1887 __ delayed()->nop();
1891 __ BIND(L_fill_elements);
1893 // in delay slot __ andcc(count, 1, G0);
1894 __ brx(Assembler::zero, false, Assembler::pt, L_fill_2);
1895 __ delayed()->andcc(count, 2, G0);
1896 __ stb(value, to, 0);
1897 __ inc(to, 1);
1898 __ BIND(L_fill_2);
1899 __ brx(Assembler::zero, false, Assembler::pt, L_fill_4);
1900 __ delayed()->andcc(count, 4, G0);
1901 __ stb(value, to, 0);
1902 __ stb(value, to, 1);
1903 __ inc(to, 2);
1904 __ BIND(L_fill_4);
1905 __ brx(Assembler::zero, false, Assembler::pt, L_exit);
1906 __ delayed()->nop();
1907 __ stb(value, to, 0);
1908 __ stb(value, to, 1);
1909 __ stb(value, to, 2);
1910 __ retl();
1911 __ delayed()->stb(value, to, 3);
1916 __ BIND(L_fill_elements);
1917 // in delay slot __ andcc(count, 1, G0);
1918 __ brx(Assembler::zero, false, Assembler::pt, L_fill_2);
1919 __ delayed()->andcc(count, 2, G0);
1920 __ sth(value, to, 0);
1921 __ inc(to, 2);
1922 __ BIND(L_fill_2);
1923 __ brx(Assembler::zero, false, Assembler::pt, L_exit);
1924 __ delayed()->nop();
1925 __ sth(value, to, 0);
1926 __ retl();
1927 __ delayed()->sth(value, to, 2);
1945 __ align(CodeEntryAlignment);
1947 address start = __ pc();
1963 *entry = __ pc();
1970 __ sllx(count, LogBytesPerShort, byte_count);
1971 __ add(to, byte_count, end_to); // offset after last copied element
1974 __ cmp(count, 11); // 8 + 3 (22 bytes)
1975 __ brx(Assembler::less, false, Assembler::pn, L_copy_2_bytes);
1976 __ delayed()->add(from, byte_count, end_from);
1983 __ andcc(end_to, 3, G0);
1984 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1985 __ delayed()->lduh(end_from, -2, O3);
1986 __ dec(end_from, 2);
1987 __ dec(end_to, 2);
1988 __ dec(count);
1989 __ sth(O3, end_to, 0);
1990 __ BIND(L_skip_alignment);
1993 __ andcc(end_to, 7, G0);
1994 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment2);
1995 __ delayed()->lduh(end_from, -2, O3);
1996 __ dec(count, 2);
1997 __ lduh(end_from, -4, O4);
1998 __ dec(end_from, 4);
1999 __ dec(end_to, 4);
2000 __ sth(O3, end_to, 2);
2001 __ sth(O4, end_to, 0);
2002 __ BIND(L_skip_alignment2);
2009 __ dec(count, 8);
2023 __ align(OptoLoopAlignment);
2024 __ BIND(L_aligned_copy);
2025 __ dec(end_from, 16);
2026 __ ldx(end_from, 8, O3);
2027 __ ldx(end_from, 0, O4);
2028 __ dec(end_to, 16);
2029 __ deccc(count, 8);
2030 __ stx(O3, end_to, 8);
2031 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
2032 __ delayed()->stx(O4, end_to, 0);
2033 __ inc(count, 8);
2036 __ BIND(L_copy_2_bytes);
2037 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
2038 __ BIND(L_copy_2_bytes_loop);
2039 __ dec(end_from, 2);
2040 __ dec(end_to, 2);
2041 __ lduh(end_from, 0, O4);
2042 __ deccc(count);
2043 __ brx(Assembler::greater, false, Assembler::pt, L_copy_2_bytes_loop);
2044 __ delayed()->sth(O4, end_to, 0);
2046 __ BIND(L_exit);
2049 __ retl();
2050 __ delayed()->mov(G0, O0); // return 0
2060 __ align(OptoLoopAlignment);
2061 __ BIND(L_loop);
2064 __ prefetch(from, ArraycopySrcPrefetchDistance, Assembler::severalReads);
2067 __ prefetch(to, ArraycopyDstPrefetchDistance, Assembler::severalWritesAndPossiblyReads);
2070 __ ldx(from, 4, O4);
2071 __ ldx(from, 12, G4);
2072 __ inc(to, 16);
2073 __ inc(from, 16);
2074 __ deccc(count, 4); // Can we do next iteration after this one?
2076 __ srlx(O4, 32, G3);
2077 __ bset(G3, O3);
2078 __ sllx(O4, 32, O4);
2079 __ srlx(G4, 32, G3);
2080 __ bset(G3, O4);
2082 __ stxa(O3, to, -16);
2083 __ stxa(O4, to, -8);
2085 __ stx(O3, to, -16);
2086 __ stx(O4, to, -8);
2088 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
2089 __ delayed()->sllx(G4, 32, O3);
2129 __ cmp(count, 5); // 4 + 1 (20 bytes)
2130 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_4_bytes);
2131 __ delayed()->mov(G0, offset);
2134 __ andcc(to, 7, G0);
2135 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
2136 __ delayed()->ld(from, 0, O3);
2137 __ inc(from, 4);
2138 __ inc(to, 4);
2139 __ dec(count);
2140 __ st(O3, to, -4);
2141 __ BIND(L_skip_alignment);
2144 __ andcc(from, 7, G0);
2145 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
2146 __ delayed()->ld(from, 0, O3);
2156 __ dec(count, 4); // The cmp at the beginning guaranty count >= 4
2157 __ sllx(O3, 32, O3);
2161 __ br(Assembler::always, false, Assembler::pt, L_copy_4_bytes);
2162 __ delayed()->inc(count, 4); // restore 'count'
2164 __ BIND(L_aligned_copy);
2168 __ and3(count, 1, G4); // Save
2169 __ srl(count, 1, count);
2171 __ mov(G4, count); // Restore
2174 __ BIND(L_copy_4_bytes);
2175 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
2176 __ BIND(L_copy_4_bytes_loop);
2177 __ ld(from, offset, O3);
2178 __ deccc(count);
2179 __ st(O3, to, offset);
2180 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_4_bytes_loop);
2181 __ delayed()->inc(offset, 4);
2182 __ BIND(L_exit);
2195 __ align(CodeEntryAlignment);
2197 address start = __ pc();
2203 *entry = __ pc();
2212 __ retl();
2213 __ delayed()->mov(G0, O0); // return 0
2242 __ sllx(count, LogBytesPerInt, byte_count);
2243 __ add(to, byte_count, end_to); // offset after last copied element
2245 __ cmp(count, 5); // for short arrays, just do single element copy
2246 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_4_bytes);
2247 __ delayed()->add(from, byte_count, end_from);
2250 __ andcc(end_to, 7, G0);
2251 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
2252 __ delayed()->nop();
2253 __ dec(count);
2254 __ dec(end_from, 4);
2255 __ dec(end_to, 4);
2256 __ ld(end_from, 0, O4);
2257 __ st(O4, end_to, 0);
2258 __ BIND(L_skip_alignment);
2261 __ andcc(end_from, 7, G0);
2262 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
2263 __ delayed()->dec(count, 4); // The cmp at the start guaranty cnt >= 4
2270 __ ldx(end_from, -4, O3);
2271 __ align(OptoLoopAlignment);
2272 __ BIND(L_copy_16_bytes);
2273 __ ldx(end_from, -12, O4);
2274 __ deccc(count, 4);
2275 __ ldx(end_from, -20, O5);
2276 __ dec(end_to, 16);
2277 __ dec(end_from, 16);
2278 __ srlx(O3, 32, O3);
2279 __ sllx(O4, 32, G3);
2280 __ bset(G3, O3);
2281 __ stx(O3, end_to, 8);
2282 __ srlx(O4, 32, O4);
2283 __ sllx(O5, 32, G3);
2284 __ bset(O4, G3);
2285 __ stx(G3, end_to, 0);
2286 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
2287 __ delayed()->mov(O5, O3);
2289 __ br(Assembler::always, false, Assembler::pt, L_copy_4_bytes);
2290 __ delayed()->inc(count, 4);
2293 __ align(OptoLoopAlignment);
2294 __ BIND(L_aligned_copy);
2295 __ dec(end_from, 16);
2296 __ ldx(end_from, 8, O3);
2297 __ ldx(end_from, 0, O4);
2298 __ dec(end_to, 16);
2299 __ deccc(count, 4);
2300 __ stx(O3, end_to, 8);
2301 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
2302 __ delayed()->stx(O4, end_to, 0);
2303 __ inc(count, 4);
2306 __ BIND(L_copy_4_bytes);
2307 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
2308 __ BIND(L_copy_4_bytes_loop);
2309 __ dec(end_from, 4);
2310 __ dec(end_to, 4);
2311 __ ld(end_from, 0, O4);
2312 __ deccc(count);
2313 __ brx(Assembler::greater, false, Assembler::pt, L_copy_4_bytes_loop);
2314 __ delayed()->st(O4, end_to, 0);
2315 __ BIND(L_exit);
2329 __ align(CodeEntryAlignment);
2331 address start = __ pc();
2336 *entry = __ pc();
2347 __ retl();
2348 __ delayed()->mov(G0, O0); // return 0
2357 __ align(OptoLoopAlignment);
2358 __ BIND(L_loop);
2362 __ prefetch(from, ArraycopySrcPrefetchDistance+off, Assembler::severalReads);
2365 __ prefetch(to, ArraycopyDstPrefetchDistance+off, Assembler::severalWritesAndPossiblyReads);
2368 __ ldx(from, off+0, O4);
2369 __ ldx(from, off+8, O5);
2371 __ stxa(O4, to, off+0);
2372 __ stxa(O5, to, off+8);
2374 __ stx(O4, to, off+0);
2375 __ stx(O5, to, off+8);
2378 __ deccc(count, 8);
2379 __ inc(from, 64);
2380 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
2381 __ delayed()->inc(to, 64);
2423 __ deccc(count, 2);
2424 __ mov(G0, offset0); // offset from start of arrays (0)
2425 __ brx(Assembler::negative, false, Assembler::pn, L_copy_8_bytes );
2426 __ delayed()->add(offset0, 8, offset8);
2432 __ subcc(count, 6, O3);
2433 __ brx(Assembler::negative, false, Assembler::pt, L_copy_16_bytes );
2434 __ delayed()->mov(to, to64);
2436 __ mov(O3, count);
2438 __ mov(from, from64);
2443 __ sub(from64, from, offset0);
2444 __ inccc(count, 6); // restore count
2445 __ brx(Assembler::negative, false, Assembler::pn, L_copy_8_bytes );
2446 __ delayed()->add(offset0, 8, offset8);
2449 __ align(OptoLoopAlignment);
2450 __ BIND(L_copy_16_bytes);
2451 __ ldx(from, offset0, O3);
2452 __ ldx(from, offset8, G3);
2453 __ deccc(count, 2);
2454 __ stx(O3, to, offset0);
2455 __ inc(offset0, 16);
2456 __ stx(G3, to, offset8);
2457 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
2458 __ delayed()->inc(offset8, 16);
2461 __ BIND(L_copy_8_bytes);
2462 __ inccc(count, 2);
2463 __ brx(Assembler::zero, true, Assembler::pn, L_exit );
2464 __ delayed()->mov(offset0, offset8); // Set O5 used by other stubs
2465 __ ldx(from, offset0, O3);
2466 __ stx(O3, to, offset0);
2467 __ BIND(L_exit);
2481 __ align(CodeEntryAlignment);
2483 address start = __ pc();
2488 *entry = __ pc();
2497 __ retl();
2498 __ delayed()->mov(G0, O0); // return 0
2521 __ subcc(count, 1, count);
2522 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_8_bytes );
2523 __ delayed()->sllx(count, LogBytesPerLong, offset8);
2524 __ sub(offset8, 8, offset0);
2525 __ align(OptoLoopAlignment);
2526 __ BIND(L_copy_16_bytes);
2527 __ ldx(from, offset8, O2);
2528 __ ldx(from, offset0, O3);
2529 __ stx(O2, to, offset8);
2530 __ deccc(offset8, 16); // use offset8 as counter
2531 __ stx(O3, to, offset0);
2532 __ brx(Assembler::greater, false, Assembler::pt, L_copy_16_bytes);
2533 __ delayed()->dec(offset0, 16);
2535 __ BIND(L_copy_8_bytes);
2536 __ brx(Assembler::negative, false, Assembler::pn, L_exit );
2537 __ delayed()->nop();
2538 __ ldx(from, 0, O3);
2539 __ stx(O3, to, 0);
2540 __ BIND(L_exit);
2554 __ align(CodeEntryAlignment);
2556 address start = __ pc();
2563 *entry = __ pc();
2574 __ retl();
2575 __ delayed()->mov(G0, O0); // return 0
2594 __ align(CodeEntryAlignment);
2596 address start = __ pc();
2601 *entry = __ pc();
2607 __ mov(to, G1);
2608 __ mov(count, G5);
2625 __ retl();
2626 __ delayed()->mov(G0, O0); // return 0
2646 __ align(CodeEntryAlignment);
2648 address start = __ pc();
2653 *entry = __ pc();
2661 __ mov(to, G1);
2662 __ mov(count, G5);
2680 __ retl();
2681 __ delayed()->mov(G0, O0); // return 0
2701 __ check_klass_subtype_fast_path(sub_klass, super_klass, temp, noreg,
2706 __ save_frame(0);
2707 __ check_klass_subtype_slow_path(sub_klass->after_save(),
2711 __ ba(L_success);
2712 __ delayed()->restore();
2714 __ bind(L_pop_to_miss);
2715 __ restore();
2718 __ BIND(L_miss);
2746 __ align(CodeEntryAlignment);
2748 address start = __ pc();
2753 __ save_frame(0);
2754 __ restore();
2763 __ mov(O3, G1); // spill: overlap test smashes O3
2764 __ mov(O4, G4); // spill: overlap test smashes O4
2766 __ stop("checkcast_copy within a single array");
2767 __ bind(L);
2768 __ mov(G1, O3);
2769 __ mov(G4, O4);
2774 *entry = __ pc();
2781 __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it
2782 __ brx(Assembler::notZero, false, Assembler::pt, load_element);
2783 __ delayed()->mov(G0, O5_offset); // offset from start of arrays
2787 __ retl();
2788 __ delayed()->set(0, O0); // return 0 on (trivial) success
2796 __ align(OptoLoopAlignment);
2798 __ BIND(store_element);
2799 __ deccc(G1_remain); // decrement the count
2800 __ store_heap_oop(G3_oop, O1_to, O5_offset); // store the oop
2801 __ inc(O5_offset, heapOopSize); // step to next offset
2802 __ brx(Assembler::zero, true, Assembler::pt, do_card_marks);
2803 __ delayed()->set(0, O0); // return -1 on success
2806 __ BIND(load_element);
2807 __ load_heap_oop(O0_from, O5_offset, G3_oop); // load the oop
2808 __ br_null_short(G3_oop, Assembler::pt, store_element);
2810 __ load_klass(G3_oop, G4_klass); // query the object klass
2821 __ BIND(fail);
2822 __ subcc(O2_count, G1_remain, O2_count);
2823 __ brx(Assembler::zero, false, Assembler::pt, done);
2824 __ delayed()->not1(O2_count, O0); // report (-1^K) to caller
2826 __ BIND(do_card_marks);
2829 __ BIND(done);
2831 __ retl();
2832 __ delayed()->nop(); // return value in 00
2862 __ align(CodeEntryAlignment);
2864 address start = __ pc();
2869 __ or3(O0_from, O1_to, G1_bits);
2870 __ or3(O2_count, G1_bits, G1_bits);
2872 __ btst(BytesPerLong-1, G1_bits);
2873 __ br(Assembler::zero, true, Assembler::pt,
2876 __ delayed()->srax(O2_count, LogBytesPerLong, O2_count);
2878 __ btst(BytesPerInt-1, G1_bits);
2879 __ br(Assembler::zero, true, Assembler::pt,
2882 __ delayed()->srax(O2_count, LogBytesPerInt, O2_count);
2884 __ btst(BytesPerShort-1, G1_bits);
2885 __ br(Assembler::zero, true, Assembler::pt,
2888 __ delayed()->srax(O2_count, LogBytesPerShort, O2_count);
2890 __ br(Assembler::always, false, Assembler::pt,
2892 __ delayed()->nop();
2916 __ add(length, src_pos, end_pos); // src_pos + length
2917 __ lduw(src, arrayOopDesc::length_offset_in_bytes(), array_length);
2918 __ cmp(end_pos, array_length);
2919 __ br(Assembler::greater, false, Assembler::pn, L_failed);
2922 __ delayed()->add(length, dst_pos, end_pos); // dst_pos + length
2923 __ lduw(dst, arrayOopDesc::length_offset_in_bytes(), array_length);
2924 __ cmp(end_pos, array_length);
2925 __ br(Assembler::greater, false, Assembler::pn, L_failed);
2929 __ delayed()->signx(src_pos, src_pos);
2930 __ signx(dst_pos, dst_pos);
2972 __ align(CodeEntryAlignment);
2974 address start = __ pc();
2999 __ br_null(src, false, Assembler::pn, L_failed);
3002 __ delayed()->tst(src_pos);
3003 __ br(Assembler::negative, false, Assembler::pn, L_failed);
3004 __ delayed()->nop();
3007 __ br_null(dst, false, Assembler::pn, L_failed);
3010 __ delayed()->tst(dst_pos);
3011 __ br(Assembler::negative, false, Assembler::pn, L_failed);
3014 __ delayed()->tst(length);
3015 __ br(Assembler::negative, false, Assembler::pn, L_failed);
3020 __ delayed()->nop(); // ??? not good
3021 __ load_klass(src, G3_src_klass);
3023 __ delayed()->ld_ptr(src, oopDesc::klass_offset_in_bytes(), G3_src_klass);
3030 __ br_notnull_short(G3_src_klass, Assembler::pt, L_b); // it is broken if klass is NULL
3031 __ bind(L_a);
3032 __ stop("broken null klass");
3033 __ bind(L_b);
3034 __ load_klass(dst, G4_dst_klass);
3035 __ br_null(G4_dst_klass, false, Assembler::pn, L_a); // this would be broken also
3036 __ delayed()->mov(G0, G4_dst_klass); // scribble the temp
3052 __ lduw(G3_src_klass, lh_offset, G5_lh);
3055 __ load_klass(dst, G4_dst_klass);
3059 __ set(objArray_lh, O5_temp);
3060 __ cmp(G5_lh, O5_temp);
3061 __ br(Assembler::equal, false, Assembler::pt, L_objArray);
3063 __ delayed()->nop();
3065 __ delayed()->ld_ptr(dst, oopDesc::klass_offset_in_bytes(), G4_dst_klass);
3069 __ cmp_and_brx_short(G3_src_klass, G4_dst_klass, Assembler::notEqual, Assembler::pn, L_failed);
3072 __ cmp(G5_lh, Klass::_lh_neutral_value); // < 0
3073 __ br(Assembler::greaterEqual, false, Assembler::pn, L_failed);
3077 __ delayed()->nop();
3080 __ set(lh_prim_tag_in_place, O5_temp);
3081 __ cmp(G5_lh, O5_temp);
3082 __ br(Assembler::greaterEqual, false, Assembler::pt, L);
3083 __ delayed()->nop();
3084 __ stop("must be a primitive array");
3085 __ bind(L);
3088 __ delayed(); // match next insn to prev branch
3103 __ srl(G5_lh, Klass::_lh_header_size_shift, G4_offset);
3104 __ and3(G4_offset, Klass::_lh_header_size_mask, G4_offset); // array_offset
3105 __ add(src, G4_offset, src); // src array offset
3106 __ add(dst, G4_offset, dst); // dst array offset
3107 __ and3(G5_lh, Klass::_lh_log2_element_size_mask, G3_elsize); // log2 element size
3118 __ sll_ptr(src_pos, G3_elsize, src_pos);
3119 __ sll_ptr(dst_pos, G3_elsize, dst_pos);
3120 __ add(src, src_pos, from); // src_addr
3121 __ add(dst, dst_pos, to); // dst_addr
3124 __ cmp(G3_elsize, 0);
3125 __ br(Assembler::equal, true, Assembler::pt, entry_jbyte_arraycopy);
3126 __ delayed()->signx(length, count); // length
3128 __ cmp(G3_elsize, LogBytesPerShort);
3129 __ br(Assembler::equal, true, Assembler::pt, entry_jshort_arraycopy);
3130 __ delayed()->signx(length, count); // length
3132 __ cmp(G3_elsize, LogBytesPerInt);
3133 __ br(Assembler::equal, true, Assembler::pt, entry_jint_arraycopy);
3134 __ delayed()->signx(length, count); // length
3137 __ cmp_and_br_short(G3_elsize, LogBytesPerLong, Assembler::equal, Assembler::pt, L);
3138 __ stop("must be long copy, but elsize is wrong");
3139 __ bind(L);
3142 __ br(Assembler::always, false, Assembler::pt, entry_jlong_arraycopy);
3143 __ delayed()->signx(length, count); // length
3146 __ BIND(L_objArray);
3151 __ cmp(G3_src_klass, G4_dst_klass); // usual case is exact equality
3152 __ brx(Assembler::notEqual, true, Assembler::pn, L_checkcast_copy);
3153 __ delayed()->lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted from below
3159 __ add(src, arrayOopDesc::base_offset_in_bytes(T_OBJECT), src); //src offset
3160 __ add(dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT), dst); //dst offset
3161 __ sll_ptr(src_pos, LogBytesPerHeapOop, src_pos);
3162 __ sll_ptr(dst_pos, LogBytesPerHeapOop, dst_pos);
3163 __ add(src, src_pos, from); // src_addr
3164 __ add(dst, dst_pos, to); // dst_addr
3165 __ BIND(L_plain_copy);
3166 __ br(Assembler::always, false, Assembler::pt, entry_oop_arraycopy);
3167 __ delayed()->signx(length, count); // length
3169 __ BIND(L_checkcast_copy);
3174 __ cmp(G5_lh, O5_temp);
3175 __ br(Assembler::notEqual, false, Assembler::pn, L_failed);
3178 __ delayed(); // match next insn to prev branch
3183 __ add(src, arrayOopDesc::base_offset_in_bytes(T_OBJECT), src); //src offset
3184 __ add(dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT), dst); //dst offset
3185 __ sll_ptr(src_pos, LogBytesPerHeapOop, src_pos);
3186 __ sll_ptr(dst_pos, LogBytesPerHeapOop, dst_pos);
3187 __ add(src, src_pos, from); // src_addr
3188 __ add(dst, dst_pos, to); // dst_addr
3189 __ signx(length, count); // length (reloaded)
3197 __ lduw(G4_dst_klass, sco_offset, sco_temp);
3205 __ ld_ptr(G4_dst_klass, ek_offset, O4); // dest elem klass
3208 __ br(Assembler::always, false, Assembler::pt, entry_checkcast_arraycopy);
3209 __ delayed()->lduw(O4, sco_offset, O3);
3212 __ BIND(L_failed);
3213 __ retl();
3214 __ delayed()->sub(G0, 1, O0); // return -1
3228 __ align(CodeEntryAlignment);
3230 address start = __ pc();
3237 __ sllx(count, LogHeapWordSize, count); // to bytes count
3239 __ bis_zeroing(to, count, temp, Ldone);
3240 __ bind(Ldone);
3241 __ retl();
3242 __ delayed()->nop();
3465 __ emit_data((intptr_t)cdesc >> 32, relocInfo::none);
3467 __ emit_data((intptr_t)cdesc, relocInfo::none);
3468 __ emit_data(++_stub_count, relocInfo::none);
3480 while ((intptr_t)(__ pc()) % icache_line_size != 0) {
3481 __ emit_data(0, relocInfo::none);
3484 while ((intptr_t)(__ pc()) % icache_half_line_size != 0) {
3485 __ nop();