Lines Matching defs:__
73 #define __ _masm->
140 address const_addr = __ float_constant(f);
143 return __ code()->consts()->start();
151 address const_addr = __ double_constant(d);
154 return __ code()->consts()->start();
162 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
166 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
170 __ fpop();
174 __ fxch(i);
178 __ fld_s(i);
182 __ ffree(i);
186 __ int3();
191 __ push_reg(opr->as_register());
193 NOT_LP64(__ push_reg(opr->as_register_hi()));
194 __ push_reg(opr->as_register_lo());
196 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
200 __ push_oop(const_opr->as_jobject());
202 __ push_jint(const_opr->as_jint());
214 __ pop_reg(opr->as_register());
234 if (! __ reachable(laddr)) {
235 __ movptr(tmp, laddr.addr());
239 return __ as_Address(laddr);
289 __ build_frame(initial_frame_size_in_bytes());
323 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
324 __ jcc(Assembler::notZero, L);
325 __ stop("locked object is NULL");
326 __ bind(L);
329 __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
330 __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
331 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
332 __ movptr(frame_map()->address_for_monitor_object(i), rbx);
346 while ((__ offset() + ic_cmp_size) % CodeEntryAlignment != 0) {
347 __ nop();
350 int offset = __ offset();
351 __ inline_cache_check(receiver, IC_Klass);
352 assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
356 __ align(CodeEntryAlignment);
365 __ movoop(reg, o);
387 __ nop();
390 address handler_base = __ start_a_stub(exception_handler_size);
401 __ invalidate_registers(false, true, true, false, true, true);
404 __ verify_not_null_oop(rax);
407 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
408 __ should_not_reach_here();
410 __ end_a_stub();
428 __ get_thread(rsi);
429 __ movptr(rax, Address(rsi, JavaThread::exception_oop_offset()));
430 __ movptr(Address(rsi, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD);
431 __ movptr(Address(rsi, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD);
433 __ bind(_unwind_handler_entry);
434 __ verify_not_null_oop(rax);
436 __ mov(rsi, rax); // Preserve the exception
444 __ unlock_object(rdi, rbx, rax, *stub->entry());
445 __ bind(*stub->continuation());
449 __ get_thread(rax);
450 __ movptr(Address(rsp, 0), rax);
451 __ movoop(Address(rsp, sizeof(void*)), method()->constant_encoding());
452 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
456 __ mov(rax, rsi); // Restore the exception
460 __ remove_frame(initial_frame_size_in_bytes());
461 __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
478 __ nop();
481 address handler_base = __ start_a_stub(deopt_handler_size);
489 InternalAddress here(__ pc());
491 __ pushptr(here.addr());
492 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
494 __ end_a_stub();
503 __ movptr (rbx, rcx); // receiver is in rcx
504 __ movptr (rax, arg1->as_register());
507 __ load_heap_oop(rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
509 __ movptr (rcx, Address(rax, java_lang_String::offset_offset_in_bytes()));
510 __ movl (rax, Address(rax, java_lang_String::count_offset_in_bytes()));
511 __ lea (rsi, Address(rsi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
513 __ movl (rax, Address(rsi, arrayOopDesc::length_offset_in_bytes()));
514 __ lea (rsi, Address(rsi, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
519 __ load_heap_oop(rdi, Address(rbx, java_lang_String::value_offset_in_bytes()));
521 __ movptr (rcx, Address(rbx, java_lang_String::offset_offset_in_bytes()));
522 __ movl (rbx, Address(rbx, java_lang_String::count_offset_in_bytes()));
523 __ lea (rdi, Address(rdi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
525 __ movl (rbx, Address(rdi, arrayOopDesc::length_offset_in_bytes()));
526 __ lea (rdi, Address(rdi, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
530 __ mov (rcx, rbx);
531 __ subptr(rbx, rax); // subtract lengths
532 __ push (rbx); // result
533 __ cmov (Assembler::lessEqual, rax, rcx);
537 __ testptr (rax, rax);
538 __ jcc (Assembler::zero, noLoop);
541 __ load_unsigned_short(rcx, Address(rdi, 0));
542 __ load_unsigned_short(rbx, Address(rsi, 0));
543 __ subl(rcx, rbx);
544 __ jcc(Assembler::notZero, haveResult);
546 __ decrement(rax); // we already tested index: skip one
547 __ jcc(Assembler::zero, noLoop);
552 __ lea(rsi, Address(rsi, rax, Address::times_2, type2aelembytes(T_CHAR)));
553 __ lea(rdi, Address(rdi, rax, Address::times_2, type2aelembytes(T_CHAR)));
554 __ negptr(rax);
559 __ align(wordSize);
560 __ bind(loop);
561 __ load_unsigned_short(rcx, Address(rdi, rax, Address::times_2, 0));
562 __ load_unsigned_short(rbx, Address(rsi, rax, Address::times_2, 0));
563 __ subl(rcx, rbx);
564 __ jcc(Assembler::notZero, haveResult);
565 __ increment(rax);
566 __ jcc(Assembler::notZero, loop);
570 __ bind(noLoop);
571 __ pop(rax);
574 __ bind(haveResult);
576 __ mov (rax, rcx); // result of call is in rax,
587 __ remove_frame(initial_frame_size_in_bytes());
597 __ lea(rscratch1, polling_page);
598 __ relocate(relocInfo::poll_return_type);
599 __ testl(rax, Address(rscratch1, 0));
601 __ testl(rax, polling_page);
603 __ ret(0);
611 int offset = __ offset();
613 __ lea(rscratch1, polling_page);
614 offset = __ offset();
616 __ testl(rax, Address(rscratch1, 0));
619 __ testl(rax, polling_page);
626 if (from_reg != to_reg) __ mov(to_reg, from_reg);
630 __ xchgptr(a, b);
642 __ movl(dest->as_register(), c->as_jint());
648 __ movptr(dest->as_register(), c->as_jint());
655 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
657 __ movptr(dest->as_register_lo(), c->as_jint_lo());
658 __ movptr(dest->as_register_hi(), c->as_jint_hi());
667 __ movoop(dest->as_register(), c->as_jobject());
675 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
677 __ movflt(dest->as_xmm_float_reg(),
684 __ fldz();
686 __ fld1();
688 __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
697 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
699 __ movdbl(dest->as_xmm_double_reg(),
706 __ fldz();
708 __ fld1();
710 __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
729 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
733 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
737 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
743 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
746 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
748 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
768 __ movl(as_Address(addr), c->as_jint_bits());
772 __ movptr(as_Address(addr), c->as_jint_bits());
779 __ movl(as_Address(addr), (int32_t)NULL_WORD);
781 __ movptr(as_Address(addr), NULL_WORD);
786 __ movoop(as_Address(addr, noreg), c->as_jobject());
789 __ movoop(rscratch1, c->as_jobject());
791 __ encode_heap_oop(rscratch1);
793 __ movl(as_Address_lo(addr), rscratch1);
796 __ movptr(as_Address_lo(addr), rscratch1);
799 __ movoop(as_Address(addr), c->as_jobject());
810 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
812 __ movptr(r10, (intptr_t)c->as_jlong_bits());
814 __ movptr(as_Address_lo(addr), r10);
818 __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
819 __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
825 __ movb(as_Address(addr), c->as_jint() & 0xFF);
830 __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
858 __ verify_oop(src->as_register());
866 __ verify_oop(src->as_register());
900 __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
901 __ fld_s(Address(rsp, 0));
903 __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
904 __ fld_d(Address(rsp, 0));
906 __ fstp_s(Address(rsp, 0));
907 __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
909 __ fstp_d(Address(rsp, 0));
910 __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
915 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
918 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
936 __ verify_oop(src->as_register());
937 __ movptr (dst, src->as_register());
939 __ movl (dst, src->as_register());
945 __ movptr (dstLO, src->as_register_lo());
946 NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
950 __ movflt(dst_addr, src->as_xmm_float_reg());
954 __ movdbl(dst_addr, src->as_xmm_double_reg());
959 if (pop_fpu_stack) __ fstp_s (dst_addr);
960 else __ fst_s (dst_addr);
965 if (pop_fpu_stack) __ fstp_d (dst_addr);
966 else __ fst_d (dst_addr);
980 __ verify_oop(src->as_register());
983 __ movptr(compressed_src, src->as_register());
984 __ encode_heap_oop(compressed_src);
999 __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
1003 if (pop_fpu_stack) __ fstp_s(as_Address(to_addr));
1004 else __ fst_s (as_Address(to_addr));
1011 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1015 if (pop_fpu_stack) __ fstp_d(as_Address(to_addr));
1016 else __ fst_d (as_Address(to_addr));
1024 __ movl(as_Address(to_addr), compressed_src);
1026 __ movptr(as_Address(to_addr), src->as_register());
1030 __ movptr(as_Address(to_addr), src->as_register());
1033 __ movl(as_Address(to_addr), src->as_register());
1040 __ movptr(as_Address_lo(to_addr), from_lo);
1050 __ movl(as_Address_hi(to_addr), from_hi);
1056 __ movl(as_Address_lo(to_addr), from_lo);
1059 __ movl(as_Address_lo(to_addr), from_lo);
1065 __ movl(as_Address_hi(to_addr), from_hi);
1076 __ movb(dst_addr, src_reg);
1082 __ movw(as_Address(to_addr), src->as_register());
1104 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1105 __ verify_oop(dest->as_register());
1107 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1113 __ movptr(dest->as_register_lo(), src_addr_LO);
1114 NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1118 __ movflt(dest->as_xmm_float_reg(), src_addr);
1122 __ movdbl(dest->as_xmm_double_reg(), src_addr);
1127 __ fld_s(src_addr);
1132 __ fld_d(src_addr);
1143 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1144 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1147 __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1148 __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1151 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1152 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1158 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1159 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1161 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1163 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1164 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1165 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1191 __ xorptr(dest->as_register(), dest->as_register());
1208 __ movflt(dest->as_xmm_float_reg(), from_addr);
1212 __ fld_s(from_addr);
1219 __ movdbl(dest->as_xmm_double_reg(), from_addr);
1223 __ fld_d(from_addr);
1231 __ movl(dest->as_register(), from_addr);
1233 __ movptr(dest->as_register(), from_addr);
1238 __ movptr(dest->as_register(), from_addr);
1241 __ movl(dest->as_register(), from_addr);
1248 __ movptr(to_lo, as_Address_lo(addr));
1261 __ lea(to_hi, as_Address(addr));
1262 __ movl(to_lo, Address(to_hi, 0));
1263 __ movl(to_hi, Address(to_hi, BytesPerWord));
1267 __ movl(to_hi, as_Address_hi(addr));
1273 __ movl(to_lo, as_Address_lo(addr));
1276 __ movl(to_lo, as_Address_lo(addr));
1282 __ movl(to_hi, as_Address_hi(addr));
1293 __ movsbl(dest_reg, from_addr);
1295 __ movb(dest_reg, from_addr);
1296 __ shll(dest_reg, 24);
1297 __ sarl(dest_reg, 24);
1306 __ movzwl(dest_reg, from_addr);
1308 __ movw(dest_reg, from_addr);
1316 __ movswl(dest_reg, from_addr);
1318 __ movw(dest_reg, from_addr);
1319 __ shll(dest_reg, 16);
1320 __ sarl(dest_reg, 16);
1336 __ decode_heap_oop(dest->as_register());
1339 __ verify_oop(dest->as_register());
1351 __ prefetchnta(from_addr); break;
1353 __ prefetcht0(from_addr); break;
1355 __ prefetcht2(from_addr); break;
1360 __ prefetchr(from_addr);
1372 __ prefetchnta(from_addr); break;
1374 __ prefetcht0(from_addr); break;
1376 __ prefetcht2(from_addr); break;
1378 __ prefetchw(from_addr); break;
1383 __ prefetchw(from_addr);
1426 __ jmp (*(op->label()));
1431 __ jcc(Assembler::parity, *(op->ublock()->label()));
1454 __ jcc(acond,*(op->label()));
1465 __ movl2ptr(dest->as_register_lo(), src->as_register());
1469 __ sarl(dest->as_register_hi(), 31);
1475 __ movl(dest->as_register(), src->as_register_lo());
1483 __ sign_extend_byte(dest->as_register());
1488 __ andl(dest->as_register(), 0xFFFF);
1493 __ sign_extend_short(dest->as_register());
1500 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1502 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1512 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1514 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1517 __ movl(Address(rsp, 0), src->as_register());
1518 __ fild_s(Address(rsp, 0));
1525 __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1527 __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1530 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1531 __ fist_s(Address(rsp, 0));
1532 __ movl(dest->as_register(), Address(rsp, 0));
1533 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1538 __ cmpl(dest->as_register(), 0x80000000);
1539 __ jcc(Assembler::equal, *op->stub()->entry());
1540 __ bind(*op->stub()->continuation());
1548 __ movptr(Address(rsp, 0), src->as_register_lo());
1549 NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1550 __ fild_d(Address(rsp, 0));
1562 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1572 __ cmpb(Address(op->klass()->as_register(),
1576 __ jcc(Assembler::notEqual, *op->stub()->entry());
1578 __ allocate_object(op->obj()->as_register(),
1585 __ bind(*op->stub()->continuation());
1590 LP64_ONLY( __ movslq(len, len); )
1595 __ jmp(*op->stub()->entry());
1607 __ mov(tmp3, len);
1609 __ allocate_array(op->obj()->as_register(),
1618 __ bind(*op->stub()->continuation());
1627 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1628 __ jccb(Assembler::notEqual, next_test);
1630 __ addptr(data_addr, DataLayout::counter_increment);
1631 __ jmp(*update_done);
1632 __ bind(next_test);
1639 __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
1640 __ jccb(Assembler::notEqual, next_test);
1641 __ movptr(recv_addr, recv);
1642 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1643 __ jmp(*update_done);
1644 __ bind(next_test);
1693 __ movoop(k_RInfo, k->constant_encoding());
1698 __ cmpptr(obj, (int32_t)NULL_WORD);
1701 __ jccb(Assembler::notEqual, not_null);
1704 __ movoop(mdo, md->constant_encoding());
1707 __ orl(data_addr, header_bits);
1708 __ jmp(*obj_is_null);
1709 __ bind(not_null);
1711 __ jcc(Assembler::equal, *obj_is_null);
1713 __ verify_oop(obj);
1720 __ load_klass(Rtmp1, obj);
1721 __ cmpptr(k_RInfo, Rtmp1);
1723 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1727 __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1729 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1732 __ jcc(Assembler::notEqual, *failure_target);
1737 __ load_klass(klass_RInfo, obj);
1741 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1743 __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1746 __ jcc(Assembler::notEqual, *failure_target);
1750 __ jcc(Assembler::equal, *success_target);
1753 __ cmpptr(klass_RInfo, k_RInfo);
1755 __ cmpoop(klass_RInfo, k->constant_encoding());
1757 __ jcc(Assembler::equal, *success_target);
1759 __ push(klass_RInfo);
1761 __ push(k_RInfo);
1763 __ pushoop(k->constant_encoding());
1765 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1766 __ pop(klass_RInfo);
1767 __ pop(klass_RInfo);
1769 __ cmpl(klass_RInfo, 0);
1770 __ jcc(Assembler::equal, *failure_target);
1775 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1776 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1777 __ push(klass_RInfo);
1778 __ push(k_RInfo);
1779 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1780 __ pop(klass_RInfo);
1781 __ pop(k_RInfo);
1783 __ cmpl(k_RInfo, 0);
1784 __ jcc(Assembler::equal, *failure_target);
1790 __ bind(profile_cast_success);
1791 __ movoop(mdo, md->constant_encoding());
1792 __ load_klass(recv, obj);
1795 __ jmp(*success);
1797 __ bind(profile_cast_failure);
1798 __ movoop(mdo, md->constant_encoding());
1800 __ subptr(counter_addr, DataLayout::counter_increment);
1801 __ jmp(*failure);
1803 __ jmp(*success);
1836 __ cmpptr(value, (int32_t)NULL_WORD);
1839 __ jccb(Assembler::notEqual, not_null);
1842 __ movoop(mdo, md->constant_encoding());
1845 __ orl(data_addr, header_bits);
1846 __ jmp(done);
1847 __ bind(not_null);
1849 __ jcc(Assembler::equal, done);
1853 __ load_klass(k_RInfo, array);
1854 __ load_klass(klass_RInfo, value);
1857 __ movptr(k_RInfo, Address(k_RInfo, objArrayKlass::element_klass_offset()));
1859 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1860 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1861 __ push(klass_RInfo);
1862 __ push(k_RInfo);
1863 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1864 __ pop(klass_RInfo);
1865 __ pop(k_RInfo);
1867 __ cmpl(k_RInfo, 0);
1868 __ jcc(Assembler::equal, *failure_target);
1873 __ bind(profile_cast_success);
1874 __ movoop(mdo, md->constant_encoding());
1875 __ load_klass(recv, value);
1878 __ jmpb(done);
1880 __ bind(profile_cast_failure);
1881 __ movoop(mdo, md->constant_encoding());
1883 __ subptr(counter_addr, DataLayout::counter_increment);
1884 __ jmp(*stub->entry());
1887 __ bind(done);
1894 __ bind(success);
1896 __ mov(dst, obj);
1904 __ bind(failure);
1905 __ xorptr(dst, dst);
1906 __ jmpb(done);
1907 __ bind(success);
1908 __ movptr(dst, 1);
1909 __ bind(done);
1925 __ lock();
1927 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1943 __ encode_heap_oop(cmpval);
1944 __ mov(rscratch1, newval);
1945 __ encode_heap_oop(rscratch1);
1947 __ lock();
1950 __ cmpxchgl(rscratch1, Address(addr, 0));
1955 __ lock();
1957 __ cmpxchgptr(newval, Address(addr, 0));
1962 __ lock();
1964 __ cmpxchgl(newval, Address(addr, 0));
1977 __ lock();
1979 __ cmpxchgq(newval, Address(addr, 0));
2014 __ cmov(ncond, result->as_register(), opr2->as_register());
2018 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2019 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2021 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2023 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2024 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2031 __ jcc (acond, skip);
2041 __ bind(skip);
2057 case lir_add: __ addl (lreg, rreg); break;
2058 case lir_sub: __ subl (lreg, rreg); break;
2059 case lir_mul: __ imull(lreg, rreg); break;
2067 case lir_add: __ addl(lreg, raddr); break;
2068 case lir_sub: __ subl(lreg, raddr); break;
2077 __ incrementl(lreg, c);
2081 __ decrementl(lreg, c);
2104 __ addptr(lreg_lo, rreg_lo);
2105 NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2108 __ subptr(lreg_lo, rreg_lo);
2109 NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2113 __ imulq(lreg_lo, rreg_lo);
2116 __ imull(lreg_hi, rreg_lo);
2117 __ imull(rreg_hi, lreg_lo);
2118 __ addl (rreg_hi, lreg_hi);
2119 __ mull (rreg_lo);
2120 __ addl (lreg_hi, rreg_hi);
2131 __ movptr(r10, (intptr_t) c);
2134 __ addptr(lreg_lo, r10);
2137 __ subptr(lreg_lo, r10);
2147 __ addptr(lreg_lo, c_lo);
2148 __ adcl(lreg_hi, c_hi);
2151 __ subptr(lreg_lo, c_lo);
2152 __ sbbl(lreg_hi, c_hi);
2170 case lir_add: __ addss(lreg, rreg); break;
2171 case lir_sub: __ subss(lreg, rreg); break;
2173 case lir_mul: __ mulss(lreg, rreg); break;
2175 case lir_div: __ divss(lreg, rreg); break;
2184 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2189 case lir_add: __ addss(lreg, raddr); break;
2190 case lir_sub: __ subss(lreg, raddr); break;
2192 case lir_mul: __ mulss(lreg, raddr); break;
2194 case lir_div: __ divss(lreg, raddr); break;
2206 case lir_add: __ addsd(lreg, rreg); break;
2207 case lir_sub: __ subsd(lreg, rreg); break;
2209 case lir_mul: __ mulsd(lreg, rreg); break;
2211 case lir_div: __ divsd(lreg, rreg); break;
2220 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2225 case lir_add: __ addsd(lreg, raddr); break;
2226 case lir_sub: __ subsd(lreg, raddr); break;
2228 case lir_mul: __ mulsd(lreg, raddr); break;
2230 case lir_div: __ divsd(lreg, raddr); break;
2252 raddr = __ as_Address(InternalAddress(const_addr));
2258 case lir_add: __ fadd_s(raddr); break;
2259 case lir_sub: __ fsub_s(raddr); break;
2261 case lir_mul: __ fmul_s(raddr); break;
2263 case lir_div: __ fdiv_s(raddr); break;
2273 __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2274 __ fmulp(left->fpu_regnrLo() + 1);
2289 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2295 case lir_add: __ fadd_d(raddr); break;
2296 case lir_sub: __ fsub_d(raddr); break;
2298 case lir_mul: __ fmul_d(raddr); break;
2300 case lir_div: __ fdiv_d(raddr); break;
2307 __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2308 __ fmulp(dest->fpu_regnrLo() + 1);
2326 case lir_add: __ addl(laddr, rreg); break;
2327 case lir_sub: __ subl(laddr, rreg); break;
2334 __ incrementl(laddr, c);
2338 __ decrementl(laddr, c);
2363 if (pop_fpu_stack) __ faddp(non_tos_index);
2364 else if (dest_is_tos) __ fadd (non_tos_index);
2365 else __ fadda(non_tos_index);
2370 if (pop_fpu_stack) __ fsubrp(non_tos_index);
2371 else if (dest_is_tos) __ fsub (non_tos_index);
2372 else __ fsubra(non_tos_index);
2374 if (pop_fpu_stack) __ fsubp (non_tos_index);
2375 else if (dest_is_tos) __ fsubr (non_tos_index);
2376 else __ fsuba (non_tos_index);
2382 if (pop_fpu_stack) __ fmulp(non_tos_index);
2383 else if (dest_is_tos) __ fmul (non_tos_index);
2384 else __ fmula(non_tos_index);
2390 if (pop_fpu_stack) __ fdivrp(non_tos_index);
2391 else if (dest_is_tos) __ fdiv (non_tos_index);
2392 else __ fdivra(non_tos_index);
2394 if (pop_fpu_stack) __ fdivp (non_tos_index);
2395 else if (dest_is_tos) __ fdivr (non_tos_index);
2396 else __ fdiva (non_tos_index);
2402 __ fremr(noreg);
2417 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2419 __ andpd(dest->as_xmm_double_reg(),
2424 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2432 case lir_log : __ flog() ; break;
2433 case lir_log10 : __ flog10() ; break;
2434 case lir_abs : __ fabs() ; break;
2435 case lir_sqrt : __ fsqrt(); break;
2438 __ trigfunc('s', op->as_Op2()->fpu_stack_size());
2443 __ trigfunc('c', op->as_Op2()->fpu_stack_size());
2447 __ trigfunc('t', op->as_Op2()->fpu_stack_size());
2450 __ exp_with_fallback(op->as_Op2()->fpu_stack_size());
2453 __ pow_with_fallback(op->as_Op2()->fpu_stack_size());
2469 case lir_logic_and: __ andl (reg, val); break;
2470 case lir_logic_or: __ orl (reg, val); break;
2471 case lir_logic_xor: __ xorl (reg, val); break;
2478 case lir_logic_and: __ andl (reg, raddr); break;
2479 case lir_logic_or: __ orl (reg, raddr); break;
2480 case lir_logic_xor: __ xorl (reg, raddr); break;
2486 case lir_logic_and: __ andptr (reg, rright); break;
2487 case lir_logic_or : __ orptr (reg, rright); break;
2488 case lir_logic_xor: __ xorptr (reg, rright); break;
2498 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2501 __ andq(l_lo, rscratch1);
2504 __ orq(l_lo, rscratch1);
2507 __ xorq(l_lo, rscratch1);
2516 __ andl(l_lo, r_lo);
2517 __ andl(l_hi, r_hi);
2520 __ orl(l_lo, r_lo);
2521 __ orl(l_hi, r_hi);
2524 __ xorl(l_lo, r_lo);
2525 __ xorl(l_hi, r_hi);
2545 __ andptr(l_lo, r_lo);
2546 NOT_LP64(__ andptr(l_hi, r_hi);)
2549 __ orptr(l_lo, r_lo);
2550 NOT_LP64(__ orptr(l_hi, r_hi);)
2553 __ xorptr(l_lo, r_lo);
2554 NOT_LP64(__ xorptr(l_hi, r_hi);)
2599 __ cdql(); // sign extend into rdx:rax
2601 __ subl(lreg, rdx);
2603 __ andl(rdx, divisor - 1);
2604 __ addl(lreg, rdx);
2606 __ sarl(lreg, log2_intptr(divisor));
2610 __ mov(dreg, lreg);
2611 __ andl(dreg, 0x80000000 | (divisor - 1));
2612 __ jcc(Assembler::positive, done);
2613 __ decrement(dreg);
2614 __ orl(dreg, ~(divisor - 1));
2615 __ increment(dreg);
2616 __ bind(done);
2628 int idivl_offset = __ corrected_idivl(rreg);
2645 __ cmpptr(reg1, opr2->as_register());
2648 __ cmpl(reg1, opr2->as_register());
2653 __ cmpptr(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2655 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2661 __ cmpl(reg1, c->as_jint());
2666 __ cmpptr(reg1, (int32_t)NULL_WORD);
2669 __ movoop(rscratch1, o);
2670 __ cmpptr(reg1, rscratch1);
2672 __ cmpoop(reg1, c->as_jobject());
2683 __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2693 __ cmpptr(xlo, opr2->as_register_lo());
2698 __ subl(xlo, ylo);
2699 __ sbbl(xhi, yhi);
2701 __ orl(xhi, xlo);
2708 __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2711 __ orl(xhi, xlo);
2721 __ ucomiss(reg1, opr2->as_xmm_float_reg());
2724 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2727 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2733 __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2742 __ ucomisd(reg1, opr2->as_xmm_double_reg());
2745 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2748 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2754 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2762 __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2769 __ movoop(rscratch1, c->as_jobject());
2778 __ cmpl(as_Address(addr), c->as_jint());
2783 __ cmpptr(rscratch1, as_Address(addr, noreg));
2785 __ cmpoop(as_Address(addr), c->as_jobject());
2800 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2803 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2810 __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2818 __ cmpptr(left->as_register_lo(), right->as_register_lo());
2819 __ movl(dest, -1);
2820 __ jccb(Assembler::less, done);
2821 __ set_byte_if_not_zero(dest);
2822 __ movzbl(dest, dest);
2823 __ bind(done);
2825 __ lcmp2int(left->as_register_hi(),
2838 int offset = __ offset();
2852 __ nop();
2859 assert(!os::is_MP() || (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2861 __ call(AddressLiteral(op->addr(), rtype));
2868 __ movoop(IC_Klass, (jobject)Universe::non_oop_word());
2870 (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2872 __ call(AddressLiteral(op->addr(), rh));
2884 address call_pc = __ pc();
2885 address stub = __ start_a_stub(call_stub_size);
2891 int start = __ offset();
2894 int offset = __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset;
2896 __ nop();
2899 __ relocate(static_stub_Relocation::spec(call_pc));
2900 __ movoop(rbx, (jobject)NULL);
2902 assert(!os::is_MP() || ((__ offset() + 1) % BytesPerWord) == 0, "must be aligned on MP");
2904 __ jump(RuntimeAddress(__ pc()));
2906 assert(__ offset() - start <= call_stub_size, "stub too big");
2907 __ end_a_stub();
2922 int pc_for_athrow_offset = __ offset();
2923 InternalAddress pc_for_athrow(__ pc());
2924 __ lea(exceptionPC->as_register(), pc_for_athrow);
2927 __ verify_not_null_oop(rax);
2934 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2937 __ nop();
2944 __ jmp(_unwind_handler_entry);
2963 case lir_shl: __ shll(value); break;
2964 case lir_shr: __ sarl(value); break;
2965 case lir_ushr: __ shrl(value); break;
2974 case lir_shl: __ shlptr(lo); break;
2975 case lir_shr: __ sarptr(lo); break;
2976 case lir_ushr: __ shrptr(lo); break;
2982 case lir_shl: __ lshl(hi, lo); break;
2983 case lir_shr: __ lshr(hi, lo, true); break;
2984 case lir_ushr: __ lshr(hi, lo, false); break;
3002 case lir_shl: __ shll(value, count); break;
3003 case lir_shr: __ sarl(value, count); break;
3004 case lir_ushr: __ shrl(value, count); break;
3017 case lir_shl: __ shlptr(value, count); break;
3018 case lir_shr: __ sarptr(value, count); break;
3019 case lir_ushr: __ shrptr(value, count); break;
3033 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3041 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3049 __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
3099 __ mov(c_rarg0, j_rarg0);
3101 __ mov(c_rarg1, j_rarg1);
3103 __ mov(c_rarg2, j_rarg2);
3105 __ mov(c_rarg3, j_rarg3);
3108 __ subptr(rsp, 6*wordSize);
3111 __ call(RuntimeAddress(C_entry));
3115 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3118 __ call(RuntimeAddress(copyfunc_addr));
3120 __ addptr(rsp, 6*wordSize);
3122 __ mov(c_rarg4, j_rarg4);
3124 __ call(RuntimeAddress(C_entry));
3128 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3131 __ call(RuntimeAddress(copyfunc_addr));
3135 __ push(length);
3136 __ push(dst_pos);
3137 __ push(dst);
3138 __ push(src_pos);
3139 __ push(src);
3142 __ call_VM_leaf(C_entry, 5); // removes pushed parameter from the stack
3146 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3149 __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3154 __ cmpl(rax, 0);
3155 __ jcc(Assembler::equal, *stub->continuation());
3158 __ mov(tmp, rax);
3159 __ xorl(tmp, -1);
3164 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3165 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3166 __ movptr (length, Address(rsp, 2*BytesPerWord));
3167 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3168 __ movptr (src, Address(rsp, 4*BytesPerWord));
3171 __ subl(length, tmp);
3172 __ addl(src_pos, tmp);
3173 __ addl(dst_pos, tmp);
3175 __ jmp(*stub->entry());
3177 __ bind(*stub->continuation());
3217 __ testptr(src, src);
3218 __ jcc(Assembler::zero, *stub->entry());
3221 __ testptr(dst, dst);
3222 __ jcc(Assembler::zero, *stub->entry());
3227 __ testl(src_pos, src_pos);
3228 __ jcc(Assembler::less, *stub->entry());
3231 __ testl(dst_pos, dst_pos);
3232 __ jcc(Assembler::less, *stub->entry());
3236 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3237 __ cmpl(tmp, src_length_addr);
3238 __ jcc(Assembler::above, *stub->entry());
3241 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3242 __ cmpl(tmp, dst_length_addr);
3243 __ jcc(Assembler::above, *stub->entry());
3247 __ testl(length, length);
3248 __ jcc(Assembler::less, *stub->entry());
3249 __ jcc(Assembler::zero, *stub->continuation());
3253 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3254 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3262 __ movl(tmp, src_klass_addr);
3263 __ cmpl(tmp, dst_klass_addr);
3265 __ movptr(tmp, src_klass_addr);
3266 __ cmpptr(tmp, dst_klass_addr);
3268 __ jcc(Assembler::notEqual, *stub->entry());
3274 __ push(src);
3275 __ push(dst);
3277 __ load_klass(src, src);
3278 __ load_klass(dst, dst);
3280 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
3282 __ push(src);
3283 __ push(dst);
3284 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3285 __ pop(dst);
3286 __ pop(src);
3288 __ cmpl(src, 0);
3289 __ jcc(Assembler::notEqual, cont);
3291 __ bind(slow);
3292 __ pop(dst);
3293 __ pop(src);
3306 __ load_klass(tmp, src);
3308 __ load_klass(tmp, dst);
3313 __ cmpl(klass_lh_addr, objArray_lh);
3314 __ jcc(Assembler::notEqual, *stub->entry());
3326 __ movptr(tmp, dst_klass_addr);
3327 __ movptr(tmp, Address(tmp, objArrayKlass::element_klass_offset()));
3328 __ push(tmp);
3329 __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3330 __ push(tmp);
3331 __ push(length);
3332 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3333 __ push(tmp);
3334 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3335 __ push(tmp);
3337 __ call_VM_leaf(copyfunc_addr, 5);
3339 __ movl2ptr(length, length); //higher 32bits must be null
3341 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3343 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3346 __ mov(c_rarg2, length);
3351 __ subptr(rsp, 6*wordSize);
3352 __ load_klass(c_rarg3, dst);
3353 __ movptr(c_rarg3, Address(c_rarg3, objArrayKlass::element_klass_offset()));
3355 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3356 __ call(RuntimeAddress(copyfunc_addr));
3357 __ addptr(rsp, 6*wordSize);
3359 __ load_klass(c_rarg4, dst);
3360 __ movptr(c_rarg4, Address(c_rarg4, objArrayKlass::element_klass_offset()));
3361 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3362 __ call(RuntimeAddress(copyfunc_addr));
3370 __ testl(rax, rax);
3371 __ jcc(Assembler::notZero, failed);
3372 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
3373 __ bind(failed);
3377 __ testl(rax, rax);
3378 __ jcc(Assembler::zero, *stub->continuation());
3382 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
3386 __ mov(tmp, rax);
3388 __ xorl(tmp, -1);
3391 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3392 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3393 __ movptr (length, Address(rsp, 2*BytesPerWord));
3394 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3395 __ movptr (src, Address(rsp, 4*BytesPerWord));
3398 __ subl(length, tmp);
3399 __ addl(src_pos, tmp);
3400 __ addl(dst_pos, tmp);
3403 __ jmp(*stub->entry());
3405 __ bind(cont);
3406 __ pop(dst);
3407 __ pop(src);
3421 __ movoop(tmp, default_type->constant_encoding());
3424 __ encode_heap_oop(tmp);
3430 if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
3431 else __ cmpptr(tmp, dst_klass_addr);
3432 __ jcc(Assembler::notEqual, halt);
3433 if (UseCompressedOops) __ cmpl(tmp, src_klass_addr);
3434 else __ cmpptr(tmp, src_klass_addr);
3435 __ jcc(Assembler::equal, known_ok);
3437 if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
3438 else __ cmpptr(tmp, dst_klass_addr);
3439 __ jcc(Assembler::equal, known_ok);
3440 __ cmpptr(src, dst);
3441 __ jcc(Assembler::equal, known_ok);
3443 __ bind(halt);
3444 __ stop("incorrect type information in arraycopy");
3445 __ bind(known_ok);
3451 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
3457 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3459 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3460 __ mov(c_rarg2, length);
3463 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3465 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3474 __ call_VM_leaf(entry, 0);
3476 __ bind(*stub->continuation());
3485 __ jmp(*op->stub()->entry());
3493 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3500 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3504 __ bind(*op->stub()->continuation());
3520 __ movoop(mdo, md->constant_encoding());
3547 __ addptr(data_addr, DataLayout::counter_increment);
3561 __ movoop(recv_addr, known_klass->constant_encoding());
3563 __ addptr(data_addr, DataLayout::counter_increment);
3568 __ load_klass(recv, recv);
3573 __ addptr(counter_addr, DataLayout::counter_increment);
3575 __ bind(update_done);
3579 __ addptr(counter_addr, DataLayout::counter_increment);
3589 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3594 __ align(BytesPerWord);
3600 __ negl(left->as_register());
3607 __ movptr(dst, lo);
3608 __ negptr(dst);
3611 __ lneg(hi, lo);
3624 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3626 __ xorps(dest->as_xmm_float_reg(),
3631 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3633 __ xorpd(dest->as_xmm_double_reg(),
3639 __ fchs();
3651 __ lea(reg, as_Address(addr->as_address_ptr()));
3658 __ call(RuntimeAddress(dest));
3675 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3677 __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
3678 __ psrlq(src->as_xmm_double_reg(), 32);
3679 __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
3682 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3684 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3691 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3693 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3701 __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
3703 __ fistp_d(as_Address(dest->as_address_ptr()));
3711 __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
3713 __ fild_d(as_Address(src->as_address_ptr()));
3725 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3730 // __ load_fence();
3735 // __ store_fence();
3740 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3745 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3750 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3754 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3760 // __ get_thread(result_reg->as_register_lo());
3761 __ mov(result_reg->as_register(), r15_thread);
3763 __ get_thread(result_reg->as_register());
3778 __ lock();
3780 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3782 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3789 __ encode_heap_oop(obj);
3790 __ xchgl(obj, as_Address(src->as_address_ptr()));
3791 __ decode_heap_oop(obj);
3793 __ xchgptr(obj, as_Address(src->as_address_ptr()));
3796 __ xchgl(obj, as_Address(src->as_address_ptr()));
3803 __ lock();
3805 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3807 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3817 #undef __