Lines Matching defs:__
44 #define __ masm->
168 __ enter(); // rsp becomes 16-byte aligned here
169 __ push_CPU_state(); // Push a multiple of 16 bytes
173 __ subptr(rsp, 256); // Save upper half of YMM registes
174 __ vextractf128h(Address(rsp, 0),xmm0);
175 __ vextractf128h(Address(rsp, 16),xmm1);
176 __ vextractf128h(Address(rsp, 32),xmm2);
177 __ vextractf128h(Address(rsp, 48),xmm3);
178 __ vextractf128h(Address(rsp, 64),xmm4);
179 __ vextractf128h(Address(rsp, 80),xmm5);
180 __ vextractf128h(Address(rsp, 96),xmm6);
181 __ vextractf128h(Address(rsp,112),xmm7);
182 __ vextractf128h(Address(rsp,128),xmm8);
183 __ vextractf128h(Address(rsp,144),xmm9);
184 __ vextractf128h(Address(rsp,160),xmm10);
185 __ vextractf128h(Address(rsp,176),xmm11);
186 __ vextractf128h(Address(rsp,192),xmm12);
187 __ vextractf128h(Address(rsp,208),xmm13);
188 __ vextractf128h(Address(rsp,224),xmm14);
189 __ vextractf128h(Address(rsp,240),xmm15);
193 __ subptr(rsp, frame::arg_reg_save_area_bytes);
280 __ addptr(rsp, frame::arg_reg_save_area_bytes);
287 __ vinsertf128h(xmm0, Address(rsp, 0));
288 __ vinsertf128h(xmm1, Address(rsp, 16));
289 __ vinsertf128h(xmm2, Address(rsp, 32));
290 __ vinsertf128h(xmm3, Address(rsp, 48));
291 __ vinsertf128h(xmm4, Address(rsp, 64));
292 __ vinsertf128h(xmm5, Address(rsp, 80));
293 __ vinsertf128h(xmm6, Address(rsp, 96));
294 __ vinsertf128h(xmm7, Address(rsp,112));
295 __ vinsertf128h(xmm8, Address(rsp,128));
296 __ vinsertf128h(xmm9, Address(rsp,144));
297 __ vinsertf128h(xmm10, Address(rsp,160));
298 __ vinsertf128h(xmm11, Address(rsp,176));
299 __ vinsertf128h(xmm12, Address(rsp,192));
300 __ vinsertf128h(xmm13, Address(rsp,208));
301 __ vinsertf128h(xmm14, Address(rsp,224));
302 __ vinsertf128h(xmm15, Address(rsp,240));
303 __ addptr(rsp, 256);
309 __ pop_CPU_state();
311 __ pop(rbp);
323 __ movdbl(xmm0, Address(rsp, xmm0_offset_in_bytes()));
325 __ movptr(rax, Address(rsp, rax_offset_in_bytes()));
326 __ movptr(rdx, Address(rsp, rdx_offset_in_bytes()));
329 __ addptr(rsp, return_offset_in_bytes());
454 __ verify_oop(rbx);
455 __ cmpptr(Address(rbx, in_bytes(methodOopDesc::code_offset())), (int32_t)NULL_WORD);
456 __ jcc(Assembler::equal, L);
459 __ mov(r13, rsp);
463 __ movptr(rax, Address(rsp, 0));
466 __ andptr(rsp, -(StackAlignmentInBytes));
467 __ push_CPU_state();
470 __ verify_oop(rbx);
478 __ subptr(rsp, frame::arg_reg_save_area_bytes);
480 __ mov(c_rarg0, rbx);
481 __ mov(c_rarg1, rax);
482 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
486 __ addptr(rsp, frame::arg_reg_save_area_bytes);
489 __ pop_CPU_state();
491 __ mov(rsp, r13);
492 __ bind(L);
509 __ bind(skip_fixup);
522 __ pop(rax);
525 __ mov(r13, rsp);
527 __ subptr(rsp, extraspace);
530 __ movptr(Address(rsp, 0), rax);
567 __ movl(rax, Address(rsp, ld_off));
568 __ movptr(Address(rsp, st_off), rax);
572 __ movq(rax, Address(rsp, ld_off));
579 __ movq(Address(rsp, next_off), rax);
582 __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
583 __ movptr(Address(rsp, st_off), rax);
586 __ movq(Address(rsp, st_off), rax);
594 __ movl(Address(rsp, st_off), r);
602 __ mov64(rax, CONST64(0xdeadffffdeadaaab));
603 __ movptr(Address(rsp, st_off), rax);
605 __ movq(Address(rsp, next_off), r);
607 __ movptr(Address(rsp, st_off), r);
614 __ movflt(Address(rsp, st_off), r_1->as_XMMRegister());
618 __ mov64(rax, CONST64(0xdeadffffdeadaaac));
619 __ movptr(Address(rsp, st_off), rax);
621 __ movdbl(Address(rsp, next_off), r_1->as_XMMRegister());
627 __ movptr(rcx, Address(rbx, in_bytes(methodOopDesc::interpreter_entry_offset())));
628 __ jmp(rcx);
635 __ lea(temp_reg, ExternalAddress(code_start));
636 __ cmpptr(pc_reg, temp_reg);
637 __ jcc(Assembler::belowEqual, L_fail);
638 __ lea(temp_reg, ExternalAddress(code_end));
639 __ cmpptr(pc_reg, temp_reg);
640 __ jcc(Assembler::below, L_ok);
641 __ bind(L_fail);
677 __ movptr(rax, Address(rsp, 0));
685 __ block_comment("verify_i2c { ");
700 __ block_comment(msg);
701 __ stop(msg);
702 __ bind(L_ok);
703 __ block_comment("} verify_i2ce ");
708 __ movptr(r11, rsp);
722 __ subptr(rsp, comp_words_on_stack * wordSize);
727 __ andptr(rsp, -16);
731 __ push(rax);
735 __ movptr(saved_sp, r11);
739 __ movptr(r11, Address(rbx, in_bytes(methodOopDesc::from_compiled_offset())));
777 __ movl(r13, Address(saved_sp, ld_off));
778 __ movptr(Address(rsp, st_off), r13);
791 __ movq(r13, Address(saved_sp, offset));
793 __ movq(Address(rsp, st_off), r13);
808 __ movq(r, Address(saved_sp, offset));
811 __ movl(r, Address(saved_sp, ld_off));
815 __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
817 __ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off));
832 __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx);
837 __ mov(rax, rbx);
838 __ jmp(r11);
848 address i2c_entry = __ pc();
861 address c2i_unverified_entry = __ pc();
870 __ verify_oop(holder);
871 __ load_klass(temp, receiver);
872 __ verify_oop(temp);
874 __ cmpptr(temp, Address(holder, compiledICHolderOopDesc::holder_klass_offset()));
875 __ movptr(rbx, Address(holder, compiledICHolderOopDesc::holder_method_offset()));
876 __ jcc(Assembler::equal, ok);
877 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
879 __ bind(ok);
883 __ cmpptr(Address(rbx, in_bytes(methodOopDesc::code_offset())), (int32_t)NULL_WORD);
884 __ jcc(Assembler::equal, skip_fixup);
885 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
888 address c2i_entry = __ pc();
892 __ flush();
1016 __ movslq(rax, Address(rbp, reg2offset_in(src.first())));
1017 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1020 __ movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
1025 // __ movslq(src.first()->as_Register(), src.first()->as_Register());
1026 __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1029 // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
1031 __ movq(dst.first()->as_Register(), src.first()->as_Register());
1040 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
1041 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1044 __ movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
1048 __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1051 __ movq(dst.first()->as_Register(), src.first()->as_Register());
1081 __ cmpptr(Address(rbp, reg2offset_in(src.first())), (int32_t)NULL_WORD);
1082 __ lea(rHandle, Address(rbp, reg2offset_in(src.first())));
1084 __ cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
1112 __ movptr(Address(rsp, offset), rOop);
1117 __ cmpptr(rOop, (int32_t)NULL_WORD);
1118 __ lea(rHandle, Address(rsp, offset));
1120 __ cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
1125 __ movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
1139 __ movl(rax, Address(rbp, reg2offset_in(src.first())));
1140 __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
1144 __ movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first())));
1149 __ movflt(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1154 __ movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
1169 __ mov(dst.first()->as_Register(), src.first()->as_Register());
1173 __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1177 __ movq(dst.first()->as_Register(), Address(rbp, reg2offset_out(src.first())));
1180 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
1181 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1196 __ movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
1200 __ movdbl(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1204 __ movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_out(src.first())));
1207 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
1208 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1218 __ movflt(Address(rbp, -wordSize), xmm0);
1221 __ movdbl(Address(rbp, -wordSize), xmm0);
1225 __ movptr(Address(rbp, -wordSize), rax);
1235 __ movflt(xmm0, Address(rbp, -wordSize));
1238 __ movdbl(xmm0, Address(rbp, -wordSize));
1242 __ movptr(rax, Address(rbp, -wordSize));
1250 __ push(args[i].first()->as_Register());
1252 __ subptr(rsp, 2*wordSize);
1253 __ movdbl(Address(rsp, 0), args[i].first()->as_XMMRegister());
1261 __ pop(args[i].first()->as_Register());
1263 __ movdbl(args[i].first()->as_XMMRegister(), Address(rsp, 0));
1264 __ addptr(rsp, 2*wordSize);
1287 __ movdbl(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1289 __ movdbl(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1296 __ movq(Address(rsp, offset), in_regs[i].first()->as_Register());
1301 __ movq(in_regs[i].first()->as_Register(), Address(rsp, offset));
1322 __ movl(Address(rsp, offset), reg);
1324 __ movl(reg, Address(rsp, offset));
1340 __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1342 __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1367 __ block_comment("check GC_locker::needs_gc");
1369 __ cmp8(ExternalAddress((address)GC_locker::needs_gc_address()), false);
1370 __ jcc(Assembler::equal, cont);
1378 address the_pc = __ pc();
1379 oop_maps->add_gc_map( __ offset(), map);
1380 __ set_last_Java_frame(rsp, noreg, the_pc);
1382 __ block_comment("block_for_jni_critical");
1383 __ movptr(c_rarg0, r15_thread);
1384 __ mov(r12, rsp); // remember sp
1385 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1386 __ andptr(rsp, -16); // align stack as required by ABI
1387 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::block_for_jni_critical)));
1388 __ mov(rsp, r12); // restore sp
1389 __ reinit_heapbase();
1391 __ reset_last_Java_frame(false, true);
1396 __ bind(cont);
1407 __ xorptr(reg, reg);
1409 __ xorpd(in_regs[i].first()->as_XMMRegister(), in_regs[i].first()->as_XMMRegister());
1437 __ block_comment("unpack_array_argument {");
1448 __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
1449 __ jccb(Assembler::equal, is_null);
1450 __ lea(tmp_reg, Address(reg.first()->as_Register(), arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1453 __ movl(tmp_reg, Address(tmp_reg, arrayOopDesc::length_offset_in_bytes() -
1456 __ jmpb(done);
1457 __ bind(is_null);
1459 __ xorptr(tmp_reg, tmp_reg);
1462 __ bind(done);
1464 __ block_comment("} unpack_array_argument");
1654 __ movptr(temp_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1655 __ verify_oop(temp_reg);
1657 __ verify_oop(r->as_Register());
1692 __ movptr(member_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1711 __ movptr(receiver_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1760 intptr_t start = (intptr_t)__ pc();
1761 int vep_offset = ((intptr_t)__ pc()) - start;
1766 int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
1767 __ flush();
1789 intptr_t start = (intptr_t)__ pc();
1983 __ verify_oop(receiver);
1984 __ load_klass(rscratch1, receiver);
1985 __ cmpq(ic_reg, rscratch1);
1986 __ jcc(Assembler::equal, hit);
1988 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1991 __ align(8);
1993 __ bind(hit);
1995 int vep_offset = ((intptr_t)__ pc()) - start;
2004 __ bang_stack_with_offset(StackShadowPages*os::vm_page_size());
2007 __ fat_nop();
2011 __ enter();
2013 __ subptr(rsp, stack_size - 2*wordSize);
2016 int frame_complete = ((intptr_t)__ pc()) - start;
2021 __ mov(rax, rsp);
2022 __ andptr(rax, -16); // must be 16 byte boundary (see amd64 ABI)
2023 __ cmpptr(rax, rsp);
2024 __ jcc(Assembler::equal, L);
2025 __ stop("improperly aligned stack");
2026 __ bind(L);
2110 __ block_comment(err_msg("move %d -> %d", i, c_arg));
2114 __ mov(tmp_vmreg.first()->as_Register(), in_regs[i].first()->as_Register());
2194 __ movoop(oop_handle_reg, JNIHandles::make_local(Klass::cast(method->method_holder())->java_mirror()));
2197 __ movptr(Address(rsp, klass_offset), oop_handle_reg);
2201 __ lea(oop_handle_reg, Address(rsp, klass_offset));
2203 __ movptr(c_rarg1, oop_handle_reg);
2217 intptr_t the_pc = (intptr_t) __ pc();
2220 __ set_last_Java_frame(rsp, noreg, (address)the_pc);
2230 __ movoop(c_rarg1, JNIHandles::make_local(method()));
2231 __ call_VM_leaf(
2241 __ movoop(c_rarg1, JNIHandles::make_local(method()));
2242 __ call_VM_leaf(
2267 __ mov(oop_handle_reg, c_rarg1);
2271 __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2274 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2277 __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, lock_done, &slow_path_lock);
2281 __ movl(swap_reg, 1);
2284 __ orptr(swap_reg, Address(obj_reg, 0));
2287 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2290 __ lock();
2294 __ cmpxchgptr(lock_reg, Address(obj_reg, 0));
2295 __ jcc(Assembler::equal, lock_done);
2308 __ subptr(swap_reg, rsp);
2309 __ andptr(swap_reg, 3 - os::vm_page_size());
2312 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2313 __ jcc(Assembler::notEqual, slow_path_lock);
2317 __ bind(lock_done);
2326 __ lea(c_rarg0, Address(r15_thread, in_bytes(JavaThread::jni_environment_offset())));
2330 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native);
2332 __ call(RuntimeAddress(native_func));
2335 __ restore_cpu_control_state_after_jni();
2339 case T_BOOLEAN: __ c2bool(rax); break;
2340 case T_CHAR : __ movzwl(rax, rax); break;
2341 case T_BYTE : __ sign_extend_byte (rax); break;
2342 case T_SHORT : __ sign_extend_short(rax); break;
2363 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2368 __ membar(Assembler::Membar_mask_bits(
2376 __ serialize_memory(r15_thread, rcx);
2386 __ cmp32(ExternalAddress((address)SafepointSynchronize::address_of_state()),
2390 __ jcc(Assembler::notEqual, L);
2391 __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
2392 __ jcc(Assembler::equal, Continue);
2393 __ bind(L);
2402 __ mov(c_rarg0, r15_thread);
2403 __ mov(r12, rsp); // remember sp
2404 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2405 __ andptr(rsp, -16); // align stack as required by ABI
2407 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2409 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans_and_transition)));
2411 __ mov(rsp, r12); // restore sp
2412 __ reinit_heapbase();
2419 __ jmpb(after_transition);
2422 __ bind(Continue);
2426 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
2427 __ bind(after_transition);
2431 __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), JavaThread::stack_guard_yellow_disabled);
2432 __ jcc(Assembler::equal, reguard);
2433 __ bind(reguard_done);
2443 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2448 __ biased_locking_exit(obj_reg, old_hdr, done);
2453 __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), (int32_t)NULL_WORD);
2454 __ jcc(Assembler::equal, done);
2463 __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2465 __ movptr(old_hdr, Address(rax, 0));
2469 __ lock();
2471 __ cmpxchgptr(old_hdr, Address(obj_reg, 0));
2472 __ jcc(Assembler::notEqual, slow_path_unlock);
2475 __ bind(unlock_done);
2480 __ bind(done);
2486 __ movoop(c_rarg1, JNIHandles::make_local(method()));
2487 __ call_VM_leaf(
2493 __ reset_last_Java_frame(false, true);
2498 __ testptr(rax, rax);
2499 __ jcc(Assembler::zero, L);
2500 __ movptr(rax, Address(rax, 0));
2501 __ bind(L);
2502 __ verify_oop(rax);
2507 __ movptr(rcx, Address(r15_thread, JavaThread::active_handles_offset()));
2508 __ movptr(Address(rcx, JNIHandleBlock::top_offset_in_bytes()), (int32_t)NULL_WORD);
2513 __ leave();
2517 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2518 __ jcc(Assembler::notEqual, exception_pending);
2523 __ ret(0);
2529 __ bind(exception_pending);
2532 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2539 __ bind(slow_path_lock);
2547 __ mov(c_rarg0, obj_reg);
2548 __ mov(c_rarg1, lock_reg);
2549 __ mov(c_rarg2, r15_thread);
2552 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2557 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2558 __ jcc(Assembler::equal, L);
2559 __ stop("no pending exception allowed on exit from monitorenter");
2560 __ bind(L);
2563 __ jmp(lock_done);
2568 __ bind(slow_path_unlock);
2577 __ lea(c_rarg1, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2579 __ mov(c_rarg0, obj_reg);
2580 __ mov(r12, rsp); // remember sp
2581 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2582 __ andptr(rsp, -16); // align stack as required by ABI
2586 __ movptr(rbx, Address(r15_thread, in_bytes(Thread::pending_exception_offset())));
2587 __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2589 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C)));
2590 __ mov(rsp, r12); // restore sp
2591 __ reinit_heapbase();
2595 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int)NULL_WORD);
2596 __ jcc(Assembler::equal, L);
2597 __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
2598 __ bind(L);
2602 __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), rbx);
2607 __ jmp(unlock_done);
2615 __ bind(reguard);
2617 __ mov(r12, rsp); // remember sp
2618 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2619 __ andptr(rsp, -16); // align stack as required by ABI
2620 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
2621 __ mov(rsp, r12); // restore sp
2622 __ reinit_heapbase();
2625 __ jmp(reguard_done);
2629 __ flush();
2822 intptr_t start = (intptr_t)__ pc();
2836 __ verify_oop(receiver);
2837 __ cmpl(ic_reg, Address(receiver, oopDesc::klass_offset_in_bytes()));
2838 __ jcc(Assembler::equal, hit);
2840 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2845 __ align(8);
2847 __ bind(hit);
2849 int vep_offset = ((intptr_t)__ pc()) - start;
2860 __ bang_stack_with_offset(StackShadowPages*os::vm_page_size());
2862 __ movl(rax, stack_size);
2863 __ bang_stack_size(rax, rbx);
2867 __ fat_nop();
2870 assert(((uintptr_t)__ pc() - start - vep_offset) >= 5,
2874 __ enter();
2878 __ subq(rsp, stack_size - 2*wordSize);
2883 int frame_complete = ((intptr_t)__ pc()) - start;
2950 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
2954 __ movptr(Address(rsp, reg2offset_out(dst.first())),
2956 __ testq(in_reg, in_reg);
2957 __ jcc(Assembler::zero, skipUnbox);
2963 __ movq(in_reg, src1);
2964 __ movq(stack_dst, in_reg);
2968 __ movl(in_reg, src1);
2969 __ movl(stack_dst, in_reg);
2972 __ bind(skipUnbox);
2976 __ xorq(rax, rax);
2979 __ movq(stack_dst, rax);
3036 __ movq(utf8_addr, src.first()->as_Register());
3045 __ movflt(src_tmp, f);
3047 __ movdbl(src_tmp, f);
3058 __ movq(src_tmp, src.first()->as_Register());
3094 __ movq(c_rarg0, src->as_Register());
3096 __ movq(c_rarg0, utf8_addr);
3100 __ movq(c_rarg0, Address(rbp, reg2offset_in(src)));
3105 __ testq(c_rarg0, c_rarg0);
3106 __ jcc(Assembler::notEqual, convert);
3111 __ movq(Address(rbp, src_off), c_rarg0);
3113 __ movq(Address(rsp, reg2offset_out(dst.first())), c_rarg0);
3115 __ jmp(done);
3117 __ bind(convert);
3119 __ lea(c_rarg1, utf8_addr);
3121 __ movq(Address(rbp, src_off), c_rarg1);
3123 __ movq(Address(rsp, reg2offset_out(dst.first())), c_rarg1);
3126 __ call(RuntimeAddress(
3129 __ bind(done);
3181 __ movq(Address(rbp, fp_offset[dst.first()->value()]), r);
3187 __ movdl(r, src.first()->as_XMMRegister());
3189 __ movdq(r, src.first()->as_XMMRegister());
3192 __ movq(r, src.first()->as_Register());
3202 __ movq(r, Address(rbp, fp_offset[src.first()->value()]));
3204 __ movl(r, Address(rbp, fp_offset[src.first()->value()]));
3211 __ movq(r, Address(rbp, reg2offset_in(src.first())));
3222 __ testq(r, r);
3223 __ jcc(Assembler::equal, skip);
3228 __ movq(r, src1);
3230 __ movl(r, src1);
3232 __ bind(skip);
3236 __ xorq(r, r);
3252 int patch_offset = ((intptr_t)__ pc()) - start;
3254 __ nop();
3259 __ leave();
3260 __ ret(0);
3262 __ flush();
3325 address start = __ pc();
3334 __ movl(r14, Deoptimization::Unpack_deopt); // callee-saved
3335 __ jmp(cont);
3337 int reexecute_offset = __ pc() - start;
3345 __ movl(r14, Deoptimization::Unpack_reexecute); // callee-saved
3346 __ jmp(cont);
3348 int exception_offset = __ pc() - start;
3357 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), rdx);
3358 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), rax);
3360 int exception_in_tls_offset = __ pc() - start;
3373 __ push(0);
3381 __ movl(r14, Deoptimization::Unpack_exception); // callee-saved
3386 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
3387 __ movptr(Address(rbp, wordSize), rdx);
3388 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), (int32_t)NULL_WORD);
3392 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
3393 __ verify_oop(rax);
3397 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
3398 __ testptr(rax, rax);
3399 __ jcc(Assembler::zero, no_pending_exception);
3400 __ stop("must not have pending exception here");
3401 __ bind(no_pending_exception);
3404 __ bind(cont);
3413 __ set_last_Java_frame(noreg, noreg, NULL);
3416 __ cmpptr(Address(r15_thread,
3419 __ jcc(Assembler::equal, L);
3420 __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
3421 __ bind(L);
3424 __ mov(c_rarg0, r15_thread);
3425 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
3429 oop_maps->add_gc_map(__ pc() - start, map);
3431 __ reset_last_Java_frame(false, false);
3434 __ mov(rdi, rax);
3437 __ cmpl(r14, Deoptimization::Unpack_exception); // Was exception pending?
3438 __ jcc(Assembler::notEqual, noException);
3439 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
3441 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
3442 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), (int32_t)NULL_WORD);
3443 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), (int32_t)NULL_WORD);
3445 __ verify_oop(rax);
3448 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3450 __ movptr(Address(rsp, RegisterSaver::rdx_offset_in_bytes()), rdx);
3452 __ bind(noException);
3474 __ movl(rcx, Address(rdi, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
3475 __ addptr(rsp, rcx);
3481 __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
3482 __ bang_stack_size(rbx, rcx);
3486 __ movptr(rcx, Address(rdi, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
3489 __ addptr(rsp, wordSize);
3492 __ movptr(rsi, Address(rdi, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
3495 __ movl(rdx, Address(rdi, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
3498 __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset_in_bytes()));
3507 __ mov(sender_sp, rsp);
3508 __ movl(rbx, Address(rdi,
3511 __ subptr(rsp, rbx);
3515 __ bind(loop);
3516 __ movptr(rbx, Address(rsi, 0)); // Load frame size
3518 __ subptr(rbx, 4*wordSize); // we'll push pc and ebp by hand and
3520 __ push(0xDEADDEAD); // Make a recognizable pattern
3521 __ push(0xDEADDEAD);
3523 __ subptr(rsp, 2*wordSize); // skip the "static long no_param"
3526 __ subptr(rbx, 2*wordSize); // We'll push pc and ebp by hand
3528 __ pushptr(Address(rcx, 0)); // Save return address
3529 __ enter(); // Save old & set new ebp
3530 __ subptr(rsp, rbx); // Prolog
3532 __ movptr(Address(rbp,
3537 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD );
3538 __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize), sender_sp); // Make it walkable
3540 __ mov(sender_sp, rsp); // Pass sender_sp to next frame
3541 __ addptr(rsi, wordSize); // Bump array pointer (sizes)
3542 __ addptr(rcx, wordSize); // Bump array pointer (pcs)
3543 __ decrementl(rdx); // Decrement counter
3544 __ jcc(Assembler::notZero, loop);
3545 __ pushptr(Address(rcx, 0)); // Save final return address
3548 __ enter(); // Save old & set new ebp
3552 __ subptr(rsp, (frame_size_in_words - 2) * wordSize);
3555 __ movdbl(Address(rsp, RegisterSaver::xmm0_offset_in_bytes()), xmm0);
3556 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3567 address the_pc = __ pc();
3568 __ set_last_Java_frame(noreg, rbp, the_pc);
3570 __ andptr(rsp, -(StackAlignmentInBytes)); // Fix stack alignment as required by ABI
3571 __ mov(c_rarg0, r15_thread);
3572 __ movl(c_rarg1, r14); // second arg: exec_mode
3573 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
3575 __ movptr(rsp, Address(r15_thread, JavaThread::last_Java_sp_offset()));
3583 __ reset_last_Java_frame(true, true);
3586 __ movdbl(xmm0, Address(rsp, RegisterSaver::xmm0_offset_in_bytes()));
3587 __ movptr(rax, Address(rsp, RegisterSaver::rax_offset_in_bytes()));
3589 __ movptr(rdx, Address(rsp, RegisterSaver::rdx_offset_in_bytes()));
3592 __ leave(); // Epilog
3595 __ ret(0);
3615 address start = __ pc();
3619 __ subptr(rsp, SimpleRuntimeFrame::return_off << LogBytesPerInt); // Epilog!
3622 __ movptr(Address(rsp, SimpleRuntimeFrame::rbp_off << LogBytesPerInt), rbp);
3626 __ movl(c_rarg1, j_rarg0);
3628 __ set_last_Java_frame(noreg, noreg, NULL);
3637 __ mov(c_rarg0, r15_thread);
3638 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap)));
3646 oop_maps->add_gc_map(__ pc() - start, map);
3648 __ reset_last_Java_frame(false, false);
3651 __ mov(rdi, rax);
3661 __ addptr(rsp, (SimpleRuntimeFrame::framesize - 2) << LogBytesPerInt); // Epilog!
3664 __ movl(rcx, Address(rdi,
3667 __ addptr(rsp, rcx);
3673 __ movl(rbx, Address(rdi ,Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
3674 __ bang_stack_size(rbx, rcx);
3678 __ movptr(rcx,
3683 __ addptr(rsp, wordSize);
3686 __ movptr(rsi, Address(rdi,
3691 __ movl(rdx, Address(rdi,
3696 __ movptr(rbp,
3707 __ mov(sender_sp, rsp);
3708 __ movl(rbx, Address(rdi,
3711 __ subptr(rsp, rbx);
3715 __ bind(loop);
3716 __ movptr(rbx, Address(rsi, 0)); // Load frame size
3717 __ subptr(rbx, 2 * wordSize); // We'll push pc and rbp by hand
3718 __ pushptr(Address(rcx, 0)); // Save return address
3719 __ enter(); // Save old & set new rbp
3720 __ subptr(rsp, rbx); // Prolog
3722 __ movptr(Address(rbp,
3726 __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize),
3729 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD );
3731 __ mov(sender_sp, rsp); // Pass sender_sp to next frame
3732 __ addptr(rsi, wordSize); // Bump array pointer (sizes)
3733 __ addptr(rcx, wordSize); // Bump array pointer (pcs)
3734 __ decrementl(rdx); // Decrement counter
3735 __ jcc(Assembler::notZero, loop);
3736 __ pushptr(Address(rcx, 0)); // Save final return address
3739 __ enter(); // Save old & set new rbp
3740 __ subptr(rsp, (SimpleRuntimeFrame::framesize - 4) << LogBytesPerInt);
3746 address the_pc = __ pc();
3747 __ set_last_Java_frame(noreg, rbp, the_pc);
3756 __ andptr(rsp, -(StackAlignmentInBytes)); // Align SP as required by ABI
3757 __ mov(c_rarg0, r15_thread);
3758 __ movl(c_rarg1, Deoptimization::Unpack_uncommon_trap);
3759 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
3766 __ reset_last_Java_frame(true, true);
3769 __ leave(); // Epilog
3772 __ ret(0);
3800 address start = __ pc();
3808 __ push(rbx);
3818 __ set_last_Java_frame(noreg, noreg, NULL);
3825 __ movptr(c_rarg0, Address(r15_thread, JavaThread::saved_exception_pc_offset()));
3826 __ movptr(Address(rbp, wordSize), c_rarg0);
3830 __ mov(c_rarg0, r15_thread);
3831 __ call(RuntimeAddress(call_ptr));
3838 oop_maps->add_gc_map( __ pc() - start, map);
3842 __ reset_last_Java_frame(false, false);
3844 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3845 __ jcc(Assembler::equal, noException);
3851 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3854 __ bind(noException);
3859 __ ret(0);
3890 int start = __ offset();
3894 int frame_complete = __ offset();
3896 __ set_last_Java_frame(noreg, noreg, NULL);
3898 __ mov(c_rarg0, r15_thread);
3900 __ call(RuntimeAddress(destination));
3907 oop_maps->add_gc_map( __ offset() - start, map);
3912 __ reset_last_Java_frame(false, false);
3915 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3916 __ jcc(Assembler::notEqual, pending);
3919 __ movptr(rbx, Address(r15_thread, JavaThread::vm_result_offset()));
3920 __ movptr(Address(rsp, RegisterSaver::rbx_offset_in_bytes()), rbx);
3922 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3928 __ jmp(rax);
3932 __ bind(pending);
3938 __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), (int)NULL_WORD);
3940 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
3941 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3996 address start = __ pc();
3999 __ push(rdx);
4000 __ subptr(rsp, SimpleRuntimeFrame::return_off << LogBytesPerInt); // Prolog
4008 __ movptr(Address(rsp, SimpleRuntimeFrame::rbp_off << LogBytesPerInt), rbp);
4014 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()),rax);
4015 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), rdx);
4027 address the_pc = __ pc();
4028 __ set_last_Java_frame(noreg, noreg, the_pc);
4029 __ mov(c_rarg0, r15_thread);
4030 __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
4031 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
4043 __ reset_last_Java_frame(false, true);
4051 __ movptr(rbp, Address(rsp, SimpleRuntimeFrame::rbp_off << LogBytesPerInt));
4053 __ addptr(rsp, SimpleRuntimeFrame::return_off << LogBytesPerInt); // Epilog
4054 __ pop(rdx); // No need for exception pc anymore
4059 __ cmpl(Address(r15_thread, JavaThread::is_method_handle_return_offset()), 0);
4060 __ cmovptr(Assembler::notEqual, rsp, rbp_mh_SP_save);
4063 __ mov(r8, rax);
4066 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
4068 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
4070 __ movptr(Address(r15_thread, JavaThread::exception_handler_pc_offset()), (int)NULL_WORD);
4071 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), (int)NULL_WORD);
4074 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), (int)NULL_WORD);
4081 __ jmp(r8);