Lines Matching defs:__
61 #define __ _masm->
68 #define BLOCK_COMMENT(str) __ block_comment(str)
99 __ incrementl(ExternalAddress((address)&counter));
233 address start = __ pc();
256 __ enter();
257 __ subptr(rsp, -rsp_after_call_off * wordSize);
261 __ movptr(parameters, c_rarg5); // parameters
262 __ movptr(entry_point, c_rarg4); // entry_point
265 __ movptr(method, c_rarg3); // method
266 __ movl(result_type, c_rarg2); // result type
267 __ movptr(result, c_rarg1); // result
268 __ movptr(call_wrapper, c_rarg0); // call wrapper
271 __ movptr(rbx_save, rbx);
272 __ movptr(r12_save, r12);
273 __ movptr(r13_save, r13);
274 __ movptr(r14_save, r14);
275 __ movptr(r15_save, r15);
278 __ movdqu(xmm_save(i), as_XMMRegister(i));
284 __ movptr(rsi_save, rsi);
285 __ movptr(rdi_save, rdi);
290 __ stmxcsr(mxcsr_save);
291 __ movl(rax, mxcsr_save);
292 __ andl(rax, MXCSR_MASK); // Only check control and mask bits
294 __ cmp32(rax, mxcsr_std);
295 __ jcc(Assembler::equal, skip_ldmx);
296 __ ldmxcsr(mxcsr_std);
297 __ bind(skip_ldmx);
302 __ movptr(r15_thread, thread);
303 __ reinit_heapbase();
309 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
310 __ jcc(Assembler::equal, L);
311 __ stop("StubRoutines::call_stub: entered with pending exception");
312 __ bind(L);
319 __ movl(c_rarg3, parameter_size);
320 __ testl(c_rarg3, c_rarg3);
321 __ jcc(Assembler::zero, parameters_done);
324 __ movptr(c_rarg2, parameters); // parameter pointer
325 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1
326 __ BIND(loop);
327 __ movptr(rax, Address(c_rarg2, 0));// get parameter
328 __ addptr(c_rarg2, wordSize); // advance to next parameter
329 __ decrementl(c_rarg1); // decrement counter
330 __ push(rax); // pass parameter
331 __ jcc(Assembler::notZero, loop);
334 __ BIND(parameters_done);
335 __ movptr(rbx, method); // get methodOop
336 __ movptr(c_rarg1, entry_point); // get entry_point
337 __ mov(r13, rsp); // set sender sp
339 __ call(c_rarg1);
342 return_address = __ pc();
346 __ movptr(c_rarg0, result);
348 __ movl(c_rarg1, result_type);
349 __ cmpl(c_rarg1, T_OBJECT);
350 __ jcc(Assembler::equal, is_long);
351 __ cmpl(c_rarg1, T_LONG);
352 __ jcc(Assembler::equal, is_long);
353 __ cmpl(c_rarg1, T_FLOAT);
354 __ jcc(Assembler::equal, is_float);
355 __ cmpl(c_rarg1, T_DOUBLE);
356 __ jcc(Assembler::equal, is_double);
359 __ movl(Address(c_rarg0, 0), rax);
361 __ BIND(exit);
364 __ lea(rsp, rsp_after_call);
370 __ cmpptr(r15_thread, thread);
371 __ jcc(Assembler::notEqual, S);
372 __ get_thread(rbx);
373 __ cmpptr(r15_thread, rbx);
374 __ jcc(Assembler::equal, L);
375 __ bind(S);
376 __ jcc(Assembler::equal, L);
377 __ stop("StubRoutines::call_stub: threads must correspond");
378 __ bind(L);
385 __ movdqu(as_XMMRegister(i), xmm_save(i));
388 __ movptr(r15, r15_save);
389 __ movptr(r14, r14_save);
390 __ movptr(r13, r13_save);
391 __ movptr(r12, r12_save);
392 __ movptr(rbx, rbx_save);
395 __ movptr(rdi, rdi_save);
396 __ movptr(rsi, rsi_save);
398 __ ldmxcsr(mxcsr_save);
402 __ addptr(rsp, -rsp_after_call_off * wordSize);
405 __ pop(rbp);
406 __ ret(0);
409 __ BIND(is_long);
410 __ movq(Address(c_rarg0, 0), rax);
411 __ jmp(exit);
413 __ BIND(is_float);
414 __ movflt(Address(c_rarg0, 0), xmm0);
415 __ jmp(exit);
417 __ BIND(is_double);
418 __ movdbl(Address(c_rarg0, 0), xmm0);
419 __ jmp(exit);
438 address start = __ pc();
448 __ cmpptr(r15_thread, thread);
449 __ jcc(Assembler::notEqual, S);
450 __ get_thread(rbx);
451 __ cmpptr(r15_thread, rbx);
452 __ jcc(Assembler::equal, L);
453 __ bind(S);
454 __ stop("StubRoutines::catch_exception: threads must correspond");
455 __ bind(L);
460 __ verify_oop(rax);
462 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
463 __ lea(rscratch1, ExternalAddress((address)__FILE__));
464 __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
465 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__);
470 __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
488 address start = __ pc();
503 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
504 __ jcc(Assembler::notEqual, L);
505 __ stop("StubRoutines::forward exception: no pending exception (1)");
506 __ bind(L);
511 __ movptr(c_rarg0, Address(rsp, 0));
513 __ call_VM_leaf(CAST_FROM_FN_PTR(address,
516 __ mov(rbx, rax);
519 __ pop(rdx);
520 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
521 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
527 __ testptr(rax, rax);
528 __ jcc(Assembler::notEqual, L);
529 __ stop("StubRoutines::forward exception: no pending exception (2)");
530 __ bind(L);
538 __ verify_oop(rax);
539 __ jmp(rbx);
554 address start = __ pc();
556 __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
557 __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
558 __ ret(0);
573 address start = __ pc();
575 __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
576 __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
577 __ ret(0);
598 address start = __ pc();
600 __ movl(rax, c_rarg2);
601 if ( os::is_MP() ) __ lock();
602 __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
603 __ ret(0);
624 address start = __ pc();
626 __ movq(rax, c_rarg2);
627 if ( os::is_MP() ) __ lock();
628 __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
629 __ ret(0);
645 address start = __ pc();
647 __ movl(rax, c_rarg0);
648 if ( os::is_MP() ) __ lock();
649 __ xaddl(Address(c_rarg1, 0), c_rarg0);
650 __ addl(rax, c_rarg0);
651 __ ret(0);
667 address start = __ pc();
669 __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
670 if ( os::is_MP() ) __ lock();
671 __ xaddptr(Address(c_rarg1, 0), c_rarg0);
672 __ addptr(rax, c_rarg0);
673 __ ret(0);
685 address start = __ pc();
686 __ membar(Assembler::StoreLoad);
687 __ ret(0);
702 address start = __ pc();
704 __ enter();
705 __ movptr(rax, old_fp); // callers fp
706 __ movptr(rax, older_fp); // the frame for ps()
707 __ pop(rbp);
708 __ ret(0);
719 address start = __ pc();
721 __ movptr(rax, rsp);
722 __ addptr(rax, 8); // return address is at the top of the stack.
723 __ ret(0);
737 address start = __ pc();
743 __ push(rax);
744 __ subptr(rsp, wordSize); // allocate a temp location
745 __ stmxcsr(mxcsr_save);
746 __ movl(rax, mxcsr_save);
747 __ andl(rax, MXCSR_MASK); // Only check control and mask bits
748 __ cmpl(rax, *(int *)(StubRoutines::x86::mxcsr_std()));
749 __ jcc(Assembler::equal, ok_ret);
751 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
753 __ ldmxcsr(ExternalAddress(StubRoutines::x86::mxcsr_std()));
755 __ bind(ok_ret);
756 __ addptr(rsp, wordSize);
757 __ pop(rax);
760 __ ret(0);
769 address start = __ pc();
773 __ push(rax);
774 __ push(c_rarg3);
775 __ push(c_rarg2);
776 __ push(c_rarg1);
778 __ movl(rax, 0x7f800000);
779 __ xorl(c_rarg3, c_rarg3);
780 __ movl(c_rarg2, inout);
781 __ movl(c_rarg1, c_rarg2);
782 __ andl(c_rarg1, 0x7fffffff);
783 __ cmpl(rax, c_rarg1); // NaN? -> 0
784 __ jcc(Assembler::negative, L);
785 __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
786 __ movl(c_rarg3, 0x80000000);
787 __ movl(rax, 0x7fffffff);
788 __ cmovl(Assembler::positive, c_rarg3, rax);
790 __ bind(L);
791 __ movptr(inout, c_rarg3);
793 __ pop(c_rarg1);
794 __ pop(c_rarg2);
795 __ pop(c_rarg3);
796 __ pop(rax);
798 __ ret(0);
806 address start = __ pc();
810 __ push(rax);
811 __ push(c_rarg3);
812 __ push(c_rarg2);
813 __ push(c_rarg1);
815 __ movl(rax, 0x7f800000);
816 __ xorl(c_rarg3, c_rarg3);
817 __ movl(c_rarg2, inout);
818 __ movl(c_rarg1, c_rarg2);
819 __ andl(c_rarg1, 0x7fffffff);
820 __ cmpl(rax, c_rarg1); // NaN? -> 0
821 __ jcc(Assembler::negative, L);
822 __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
823 __ mov64(c_rarg3, 0x8000000000000000);
824 __ mov64(rax, 0x7fffffffffffffff);
825 __ cmov(Assembler::positive, c_rarg3, rax);
827 __ bind(L);
828 __ movptr(inout, c_rarg3);
830 __ pop(c_rarg1);
831 __ pop(c_rarg2);
832 __ pop(c_rarg3);
833 __ pop(rax);
835 __ ret(0);
844 address start = __ pc();
848 __ push(rax);
849 __ push(c_rarg3);
850 __ push(c_rarg2);
851 __ push(c_rarg1);
852 __ push(c_rarg0);
854 __ movl(rax, 0x7ff00000);
855 __ movq(c_rarg2, inout);
856 __ movl(c_rarg3, c_rarg2);
857 __ mov(c_rarg1, c_rarg2);
858 __ mov(c_rarg0, c_rarg2);
859 __ negl(c_rarg3);
860 __ shrptr(c_rarg1, 0x20);
861 __ orl(c_rarg3, c_rarg2);
862 __ andl(c_rarg1, 0x7fffffff);
863 __ xorl(c_rarg2, c_rarg2);
864 __ shrl(c_rarg3, 0x1f);
865 __ orl(c_rarg1, c_rarg3);
866 __ cmpl(rax, c_rarg1);
867 __ jcc(Assembler::negative, L); // NaN -> 0
868 __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
869 __ movl(c_rarg2, 0x80000000);
870 __ movl(rax, 0x7fffffff);
871 __ cmov(Assembler::positive, c_rarg2, rax);
873 __ bind(L);
874 __ movptr(inout, c_rarg2);
876 __ pop(c_rarg0);
877 __ pop(c_rarg1);
878 __ pop(c_rarg2);
879 __ pop(c_rarg3);
880 __ pop(rax);
882 __ ret(0);
891 address start = __ pc();
895 __ push(rax);
896 __ push(c_rarg3);
897 __ push(c_rarg2);
898 __ push(c_rarg1);
899 __ push(c_rarg0);
901 __ movl(rax, 0x7ff00000);
902 __ movq(c_rarg2, inout);
903 __ movl(c_rarg3, c_rarg2);
904 __ mov(c_rarg1, c_rarg2);
905 __ mov(c_rarg0, c_rarg2);
906 __ negl(c_rarg3);
907 __ shrptr(c_rarg1, 0x20);
908 __ orl(c_rarg3, c_rarg2);
909 __ andl(c_rarg1, 0x7fffffff);
910 __ xorl(c_rarg2, c_rarg2);
911 __ shrl(c_rarg3, 0x1f);
912 __ orl(c_rarg1, c_rarg3);
913 __ cmpl(rax, c_rarg1);
914 __ jcc(Assembler::negative, L); // NaN -> 0
915 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
916 __ mov64(c_rarg2, 0x8000000000000000);
917 __ mov64(rax, 0x7fffffffffffffff);
918 __ cmovq(Assembler::positive, c_rarg2, rax);
920 __ bind(L);
921 __ movq(inout, c_rarg2);
923 __ pop(c_rarg0);
924 __ pop(c_rarg1);
925 __ pop(c_rarg2);
926 __ pop(c_rarg3);
927 __ pop(rax);
929 __ ret(0);
935 __ align(CodeEntryAlignment);
937 address start = __ pc();
939 __ emit_data64( mask, relocInfo::none );
940 __ emit_data64( mask, relocInfo::none );
951 address start = __ pc();
953 __ push(0); // hole for return address-to-be
954 __ pusha(); // push registers
959 __ subptr(rsp, frame::arg_reg_save_area_bytes);
961 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
962 __ addptr(rsp, frame::arg_reg_save_area_bytes);
964 __ movptr(next_pc, rax); // stuff next address
965 __ popa();
966 __ ret(0); // jump to next address
989 address start = __ pc();
993 __ pushf();
994 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
996 __ push(r12);
999 __ push(c_rarg2);
1000 __ push(c_rarg3);
1014 __ movptr(rax, Address(rsp, oop_to_verify));
1017 __ testptr(rax, rax);
1018 __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
1020 __ movptr(c_rarg2, rax);
1021 __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
1022 __ andptr(c_rarg2, c_rarg3);
1023 __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
1024 __ cmpptr(c_rarg2, c_rarg3);
1025 __ jcc(Assembler::notZero, error);
1028 __ reinit_heapbase();
1031 __ load_klass(rax, rax); // get klass
1032 __ testptr(rax, rax);
1033 __ jcc(Assembler::zero, error); // if klass is NULL it is broken
1035 __ mov(c_rarg2, rax);
1036 __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_mask());
1037 __ andptr(c_rarg2, c_rarg3);
1038 __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_bits());
1039 __ cmpptr(c_rarg2, c_rarg3);
1040 __ jcc(Assembler::notZero, error);
1043 __ load_klass(rax, rax);
1044 __ testptr(rax, rax);
1045 __ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
1047 __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_mask());
1048 __ andptr(rax, c_rarg3);
1049 __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_bits());
1050 __ cmpptr(rax, c_rarg3);
1051 __ jcc(Assembler::notZero, error);
1054 __ bind(exit);
1055 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back
1056 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1057 __ pop(c_rarg3); // restore c_rarg3
1058 __ pop(c_rarg2); // restore c_rarg2
1059 __ pop(r12); // restore r12
1060 __ popf(); // restore flags
1061 __ ret(4 * wordSize); // pop caller saved stuff
1064 __ bind(error);
1065 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back
1066 __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1067 __ pop(c_rarg3); // get saved c_rarg3 back
1068 __ pop(c_rarg2); // get saved c_rarg2 back
1069 __ pop(r12); // get saved r12 back
1070 __ popf(); // get saved flags off stack --
1073 __ pusha(); // push registers
1087 __ movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message
1088 __ movptr(c_rarg1, Address(rsp, return_addr)); // pass return address
1089 __ movq(c_rarg2, rsp); // pass address of regs on stack
1090 __ mov(r12, rsp); // remember rsp
1091 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1092 __ andptr(rsp, -16); // align stack as required by ABI
1094 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1095 __ mov(rsp, r12); // restore rsp
1096 __ popa(); // pop registers (includes r12)
1097 __ ret(4 * wordSize); // pop caller saved stuff
1114 __ movslq(Rtmp, Rint);
1115 __ cmpq(Rtmp, Rint);
1116 __ jcc(Assembler::equal, L);
1117 __ stop("high 32-bits of int value are not 0");
1118 __ bind(L);
1145 __ cmpptr(to, from);
1146 __ lea(end_from, Address(from, count, sf, 0));
1149 __ jump_cc(Assembler::belowEqual, no_overlap);
1150 __ cmpptr(to, end_from);
1151 __ jump_cc(Assembler::aboveEqual, no_overlap);
1153 __ jcc(Assembler::belowEqual, (*NOLp));
1154 __ cmpptr(to, end_from);
1155 __ jcc(Assembler::aboveEqual, (*NOLp));
1178 __ mov(rax, r9); // r9 is also saved_rdi
1179 __ movptr(saved_rdi, rdi);
1180 __ movptr(saved_rsi, rsi);
1181 __ mov(rdi, rcx); // c_rarg0
1182 __ mov(rsi, rdx); // c_rarg1
1183 __ mov(rdx, r8); // c_rarg2
1185 __ mov(rcx, rax); // c_rarg3 (via rax)
1196 __ movptr(rdi, saved_rdi);
1197 __ movptr(rsi, saved_rsi);
1216 __ pusha(); // push registers
1220 __ xchgptr(c_rarg1, c_rarg0);
1222 __ movptr(c_rarg1, count);
1223 __ movptr(c_rarg0, addr);
1226 __ movptr(c_rarg0, addr);
1227 __ movptr(c_rarg1, count);
1229 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1230 __ popa();
1260 __ pusha(); // push registers (overkill)
1263 __ mov(c_rarg1, count);
1264 __ mov(c_rarg0, start);
1267 __ mov(c_rarg0, start);
1268 __ mov(c_rarg1, count);
1270 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
1271 __ popa();
1283 __ leaq(end, Address(start, count, TIMES_OOP, 0)); // end == start+count*oop_size
1284 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
1285 __ shrptr(start, CardTableModRefBS::card_shift);
1286 __ shrptr(end, CardTableModRefBS::card_shift);
1287 __ subptr(end, start); // end --> cards count
1290 __ mov64(scratch, disp);
1291 __ addptr(start, scratch);
1292 __ BIND(L_loop);
1293 __ movb(Address(start, count, Address::times_1), 0);
1294 __ decrement(count);
1295 __ jcc(Assembler::greaterEqual, L_loop);
1318 DEBUG_ONLY(__ stop("enter at entry label, not here"));
1320 __ align(OptoLoopAlignment);
1324 __ BIND(L_loop);
1326 __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1327 __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1328 __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1329 __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1331 __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1332 __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1333 __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1334 __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1335 __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1336 __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1337 __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1338 __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1340 __ BIND(L_copy_bytes);
1341 __ addptr(qword_count, 8);
1342 __ jcc(Assembler::lessEqual, L_loop);
1343 __ subptr(qword_count, 4); // sub(8) and add(4)
1344 __ jccb(Assembler::greater, L_end);
1347 __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1348 __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1350 __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1351 __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1352 __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1353 __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1355 __ addptr(qword_count, 4);
1356 __ BIND(L_end);
1359 __ vzeroupper();
1363 __ BIND(L_loop);
1364 __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1365 __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1366 __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1367 __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1368 __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1369 __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1370 __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1371 __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1373 __ BIND(L_copy_bytes);
1374 __ addptr(qword_count, 4);
1375 __ jcc(Assembler::lessEqual, L_loop);
1377 __ subptr(qword_count, 4);
1378 __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1394 DEBUG_ONLY(__ stop("enter at entry label, not here"));
1396 __ align(OptoLoopAlignment);
1400 __ BIND(L_loop);
1402 __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1403 __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1404 __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8, 0));
1405 __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm1);
1407 __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1408 __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1409 __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1410 __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1411 __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1412 __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1413 __ movdqu(xmm3, Address(from, qword_count, Address::times_8, 0));
1414 __ movdqu(Address(dest, qword_count, Address::times_8, 0), xmm3);
1416 __ BIND(L_copy_bytes);
1417 __ subptr(qword_count, 8);
1418 __ jcc(Assembler::greaterEqual, L_loop);
1420 __ addptr(qword_count, 4); // add(8) and sub(4)
1421 __ jccb(Assembler::less, L_end);
1424 __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1425 __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1427 __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1428 __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1429 __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 0));
1430 __ movdqu(Address(dest, qword_count, Address::times_8, 0), xmm1);
1432 __ subptr(qword_count, 4);
1433 __ BIND(L_end);
1436 __ vzeroupper();
1440 __ BIND(L_loop);
1441 __ movq(to, Address(from, qword_count, Address::times_8, 24));
1442 __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1443 __ movq(to, Address(from, qword_count, Address::times_8, 16));
1444 __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1445 __ movq(to, Address(from, qword_count, Address::times_8, 8));
1446 __ movq(Address(dest, qword_count, Address::times_8, 8), to);
1447 __ movq(to, Address(from, qword_count, Address::times_8, 0));
1448 __ movq(Address(dest, qword_count, Address::times_8, 0), to);
1450 __ BIND(L_copy_bytes);
1451 __ subptr(qword_count, 4);
1452 __ jcc(Assembler::greaterEqual, L_loop);
1454 __ addptr(qword_count, 4);
1455 __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1479 __ align(CodeEntryAlignment);
1481 address start = __ pc();
1495 __ enter(); // required for proper stackwalking of RuntimeStub frame
1499 *entry = __ pc();
1508 __ movptr(byte_count, count);
1509 __ shrptr(count, 3); // count => qword_count
1512 __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1513 __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
1514 __ negptr(qword_count); // make the count negative
1515 __ jmp(L_copy_bytes);
1518 __ BIND(L_copy_8_bytes);
1519 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1520 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1521 __ increment(qword_count);
1522 __ jcc(Assembler::notZero, L_copy_8_bytes);
1525 __ BIND(L_copy_4_bytes);
1526 __ testl(byte_count, 4);
1527 __ jccb(Assembler::zero, L_copy_2_bytes);
1528 __ movl(rax, Address(end_from, 8));
1529 __ movl(Address(end_to, 8), rax);
1531 __ addptr(end_from, 4);
1532 __ addptr(end_to, 4);
1535 __ BIND(L_copy_2_bytes);
1536 __ testl(byte_count, 2);
1537 __ jccb(Assembler::zero, L_copy_byte);
1538 __ movw(rax, Address(end_from, 8));
1539 __ movw(Address(end_to, 8), rax);
1541 __ addptr(end_from, 2);
1542 __ addptr(end_to, 2);
1545 __ BIND(L_copy_byte);
1546 __ testl(byte_count, 1);
1547 __ jccb(Assembler::zero, L_exit);
1548 __ movb(rax, Address(end_from, 8));
1549 __ movb(Address(end_to, 8), rax);
1551 __ BIND(L_exit);
1554 __ xorptr(rax, rax); // return 0
1555 __ leave(); // required for proper stackwalking of RuntimeStub frame
1556 __ ret(0);
1560 __ jmp(L_copy_4_bytes);
1582 __ align(CodeEntryAlignment);
1584 address start = __ pc();
1593 __ enter(); // required for proper stackwalking of RuntimeStub frame
1597 *entry = __ pc();
1607 __ movptr(byte_count, count);
1608 __ shrptr(count, 3); // count => qword_count
1613 __ testl(byte_count, 1);
1614 __ jcc(Assembler::zero, L_copy_2_bytes);
1615 __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1616 __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1617 __ decrement(byte_count); // Adjust for possible trailing word
1620 __ BIND(L_copy_2_bytes);
1621 __ testl(byte_count, 2);
1622 __ jcc(Assembler::zero, L_copy_4_bytes);
1623 __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1624 __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1627 __ BIND(L_copy_4_bytes);
1628 __ testl(byte_count, 4);
1629 __ jcc(Assembler::zero, L_copy_bytes);
1630 __ movl(rax, Address(from, qword_count, Address::times_8));
1631 __ movl(Address(to, qword_count, Address::times_8), rax);
1632 __ jmp(L_copy_bytes);
1635 __ BIND(L_copy_8_bytes);
1636 __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1637 __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1638 __ decrement(qword_count);
1639 __ jcc(Assembler::notZero, L_copy_8_bytes);
1643 __ xorptr(rax, rax); // return 0
1644 __ leave(); // required for proper stackwalking of RuntimeStub frame
1645 __ ret(0);
1652 __ xorptr(rax, rax); // return 0
1653 __ leave(); // required for proper stackwalking of RuntimeStub frame
1654 __ ret(0);
1679 __ align(CodeEntryAlignment);
1681 address start = __ pc();
1694 __ enter(); // required for proper stackwalking of RuntimeStub frame
1698 *entry = __ pc();
1707 __ movptr(word_count, count);
1708 __ shrptr(count, 2); // count => qword_count
1711 __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1712 __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
1713 __ negptr(qword_count);
1714 __ jmp(L_copy_bytes);
1717 __ BIND(L_copy_8_bytes);
1718 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1719 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1720 __ increment(qword_count);
1721 __ jcc(Assembler::notZero, L_copy_8_bytes);
1727 __ BIND(L_copy_4_bytes);
1728 __ testl(word_count, 2);
1729 __ jccb(Assembler::zero, L_copy_2_bytes);
1730 __ movl(rax, Address(end_from, 8));
1731 __ movl(Address(end_to, 8), rax);
1733 __ addptr(end_from, 4);
1734 __ addptr(end_to, 4);
1737 __ BIND(L_copy_2_bytes);
1738 __ testl(word_count, 1);
1739 __ jccb(Assembler::zero, L_exit);
1740 __ movw(rax, Address(end_from, 8));
1741 __ movw(Address(end_to, 8), rax);
1743 __ BIND(L_exit);
1746 __ xorptr(rax, rax); // return 0
1747 __ leave(); // required for proper stackwalking of RuntimeStub frame
1748 __ ret(0);
1752 __ jmp(L_copy_4_bytes);
1758 __ align(CodeEntryAlignment);
1760 address start = __ pc();
1768 __ enter(); // required for proper stackwalking of RuntimeStub frame
1770 __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1772 __ leave(); // required for proper stackwalking of RuntimeStub frame
1773 __ ret(0);
1794 __ align(CodeEntryAlignment);
1796 address start = __ pc();
1805 __ enter(); // required for proper stackwalking of RuntimeStub frame
1809 *entry = __ pc();
1819 __ movptr(word_count, count);
1820 __ shrptr(count, 2); // count => qword_count
1825 __ testl(word_count, 1);
1826 __ jccb(Assembler::zero, L_copy_4_bytes);
1827 __ movw(rax, Address(from, word_count, Address::times_2, -2));
1828 __ movw(Address(to, word_count, Address::times_2, -2), rax);
1831 __ BIND(L_copy_4_bytes);
1832 __ testl(word_count, 2);
1833 __ jcc(Assembler::zero, L_copy_bytes);
1834 __ movl(rax, Address(from, qword_count, Address::times_8));
1835 __ movl(Address(to, qword_count, Address::times_8), rax);
1836 __ jmp(L_copy_bytes);
1839 __ BIND(L_copy_8_bytes);
1840 __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1841 __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1842 __ decrement(qword_count);
1843 __ jcc(Assembler::notZero, L_copy_8_bytes);
1847 __ xorptr(rax, rax); // return 0
1848 __ leave(); // required for proper stackwalking of RuntimeStub frame
1849 __ ret(0);
1856 __ xorptr(rax, rax); // return 0
1857 __ leave(); // required for proper stackwalking of RuntimeStub frame
1858 __ ret(0);
1884 __ align(CodeEntryAlignment);
1886 address start = __ pc();
1900 __ enter(); // required for proper stackwalking of RuntimeStub frame
1904 *entry = __ pc();
1912 __ movq(saved_to, to);
1917 __ movptr(dword_count, count);
1918 __ shrptr(count, 1); // count => qword_count
1921 __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1922 __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
1923 __ negptr(qword_count);
1924 __ jmp(L_copy_bytes);
1927 __ BIND(L_copy_8_bytes);
1928 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1929 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1930 __ increment(qword_count);
1931 __ jcc(Assembler::notZero, L_copy_8_bytes);
1934 __ BIND(L_copy_4_bytes);
1935 __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1936 __ jccb(Assembler::zero, L_exit);
1937 __ movl(rax, Address(end_from, 8));
1938 __ movl(Address(end_to, 8), rax);
1940 __ BIND(L_exit);
1946 __ xorptr(rax, rax); // return 0
1947 __ leave(); // required for proper stackwalking of RuntimeStub frame
1948 __ ret(0);
1952 __ jmp(L_copy_4_bytes);
1975 __ align(CodeEntryAlignment);
1977 address start = __ pc();
1986 __ enter(); // required for proper stackwalking of RuntimeStub frame
1990 *entry = __ pc();
2006 __ movptr(dword_count, count);
2007 __ shrptr(count, 1); // count => qword_count
2012 __ testl(dword_count, 1);
2013 __ jcc(Assembler::zero, L_copy_bytes);
2014 __ movl(rax, Address(from, dword_count, Address::times_4, -4));
2015 __ movl(Address(to, dword_count, Address::times_4, -4), rax);
2016 __ jmp(L_copy_bytes);
2019 __ BIND(L_copy_8_bytes);
2020 __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2021 __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2022 __ decrement(qword_count);
2023 __ jcc(Assembler::notZero, L_copy_8_bytes);
2026 __ jmp(L_exit);
2030 __ xorptr(rax, rax); // return 0
2031 __ leave(); // required for proper stackwalking of RuntimeStub frame
2032 __ ret(0);
2037 __ BIND(L_exit);
2043 __ xorptr(rax, rax); // return 0
2044 __ leave(); // required for proper stackwalking of RuntimeStub frame
2045 __ ret(0);
2067 __ align(CodeEntryAlignment);
2069 address start = __ pc();
2082 __ enter(); // required for proper stackwalking of RuntimeStub frame
2087 *entry = __ pc();
2097 __ movptr(saved_count, qword_count);
2103 __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2104 __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
2105 __ negptr(qword_count);
2106 __ jmp(L_copy_bytes);
2109 __ BIND(L_copy_8_bytes);
2110 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2111 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2112 __ increment(qword_count);
2113 __ jcc(Assembler::notZero, L_copy_8_bytes);
2116 __ jmp(L_exit);
2120 __ xorptr(rax, rax); // return 0
2121 __ leave(); // required for proper stackwalking of RuntimeStub frame
2122 __ ret(0);
2129 __ BIND(L_exit);
2138 __ xorptr(rax, rax); // return 0
2139 __ leave(); // required for proper stackwalking of RuntimeStub frame
2140 __ ret(0);
2159 __ align(CodeEntryAlignment);
2161 address start = __ pc();
2169 __ enter(); // required for proper stackwalking of RuntimeStub frame
2173 *entry = __ pc();
2184 __ movptr(saved_count, qword_count);
2189 __ jmp(L_copy_bytes);
2192 __ BIND(L_copy_8_bytes);
2193 __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2194 __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2195 __ decrement(qword_count);
2196 __ jcc(Assembler::notZero, L_copy_8_bytes);
2199 __ jmp(L_exit);
2203 __ xorptr(rax, rax); // return 0
2204 __ leave(); // required for proper stackwalking of RuntimeStub frame
2205 __ ret(0);
2212 __ BIND(L_exit);
2221 __ xorptr(rax, rax); // return 0
2222 __ leave(); // required for proper stackwalking of RuntimeStub frame
2223 __ ret(0);
2241 __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, NULL,
2243 __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2246 __ BIND(L_miss);
2296 __ align(CodeEntryAlignment);
2298 address start = __ pc();
2300 __ enter(); // required for proper stackwalking of RuntimeStub frame
2307 __ stop("checkcast_copy within a single array");
2308 __ bind(L);
2317 __ movptr(ckval, Address(rsp, 6 * wordSize));
2322 *entry = __ pc();
2332 __ subptr(rsp, saved_rbp_offset * wordSize);
2333 __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2334 __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2346 __ cmpl(ckoff, Address(ckval, sco_offset));
2347 __ jcc(Assembler::equal, L);
2348 __ stop("super_check_offset inconsistent");
2349 __ bind(L);
2363 __ lea(end_from, end_from_addr);
2364 __ lea(end_to, end_to_addr);
2365 __ movptr(r14_length, length); // save a copy of the length
2367 __ negptr(count); // negate and test the length
2368 __ jcc(Assembler::notZero, L_load_element);
2371 __ xorptr(rax, rax); // return 0 on (trivial) success
2372 __ jmp(L_done);
2379 __ align(OptoLoopAlignment);
2381 __ BIND(L_store_element);
2382 __ store_heap_oop(to_element_addr, rax_oop); // store the oop
2383 __ increment(count); // increment the count toward zero
2384 __ jcc(Assembler::zero, L_do_card_marks);
2387 __ BIND(L_load_element);
2388 __ load_heap_oop(rax_oop, from_element_addr); // load the oop
2389 __ testptr(rax_oop, rax_oop);
2390 __ jcc(Assembler::zero, L_store_element);
2392 __ load_klass(r11_klass, rax_oop);// query the object klass
2402 __ addptr(r14_length, count); // K = (original - remaining) oops
2403 __ movptr(rax, r14_length); // save the value
2404 __ notptr(rax); // report (-1^K) to caller (does not affect flags)
2405 __ jccb(Assembler::notZero, L_post_barrier);
2406 __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2409 __ BIND(L_do_card_marks);
2410 __ xorptr(rax, rax); // return 0 on success
2412 __ BIND(L_post_barrier);
2416 __ BIND(L_done);
2417 __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2418 __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2421 __ leave(); // required for proper stackwalking of RuntimeStub frame
2422 __ ret(0);
2454 __ align(CodeEntryAlignment);
2456 address start = __ pc();
2458 __ enter(); // required for proper stackwalking of RuntimeStub frame
2463 __ mov(bits, from);
2464 __ orptr(bits, to);
2465 __ orptr(bits, size);
2467 __ testb(bits, BytesPerLong-1);
2468 __ jccb(Assembler::zero, L_long_aligned);
2470 __ testb(bits, BytesPerInt-1);
2471 __ jccb(Assembler::zero, L_int_aligned);
2473 __ testb(bits, BytesPerShort-1);
2474 __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2476 __ BIND(L_short_aligned);
2477 __ shrptr(size, LogBytesPerShort); // size => short_count
2478 __ jump(RuntimeAddress(short_copy_entry));
2480 __ BIND(L_int_aligned);
2481 __ shrptr(size, LogBytesPerInt); // size => int_count
2482 __ jump(RuntimeAddress(int_copy_entry));
2484 __ BIND(L_long_aligned);
2485 __ shrptr(size, LogBytesPerLong); // size => qword_count
2486 __ jump(RuntimeAddress(long_copy_entry));
2504 __ movl(temp, length);
2505 __ addl(temp, src_pos); // src_pos + length
2506 __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2507 __ jcc(Assembler::above, L_failed);
2510 __ movl(temp, length);
2511 __ addl(temp, dst_pos); // dst_pos + length
2512 __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2513 __ jcc(Assembler::above, L_failed);
2517 __ movslq(src_pos, src_pos);
2518 __ movslq(dst_pos, dst_pos);
2561 int advance = target - (__ offset() % modulus);
2563 if (advance > 0) __ nop(advance);
2568 __ BIND(L_failed_0);
2569 __ jmp(L_failed);
2570 assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2572 __ align(CodeEntryAlignment);
2573 address start = __ pc();
2575 __ enter(); // required for proper stackwalking of RuntimeStub frame
2595 __ testptr(src, src); // src oop
2596 size_t j1off = __ offset();
2597 __ jccb(Assembler::zero, L_failed_0);
2600 __ testl(src_pos, src_pos); // src_pos (32-bits)
2601 __ jccb(Assembler::negative, L_failed_0);
2604 __ testptr(dst, dst); // dst oop
2605 __ jccb(Assembler::zero, L_failed_0);
2608 __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2609 size_t j4off = __ offset();
2610 __ jccb(Assembler::negative, L_failed_0);
2625 __ movl(r11_length, length); // length (elements count, 32-bits value)
2626 __ testl(r11_length, r11_length);
2627 __ jccb(Assembler::negative, L_failed_0);
2629 __ load_klass(r10_src_klass, src);
2635 __ testptr(r10_src_klass, r10_src_klass);
2636 __ jcc(Assembler::notZero, L2); // it is broken if klass is NULL
2637 __ bind(L1);
2638 __ stop("broken null klass");
2639 __ bind(L2);
2640 __ load_klass(rax, dst);
2641 __ cmpq(rax, 0);
2642 __ jcc(Assembler::equal, L1); // this would be broken also
2659 __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2660 __ jcc(Assembler::equal, L_objArray);
2663 __ load_klass(rax, dst);
2664 __ cmpq(r10_src_klass, rax);
2665 __ jcc(Assembler::notEqual, L_failed);
2668 __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2671 __ cmpl(rax_lh, Klass::_lh_neutral_value);
2672 __ jcc(Assembler::greaterEqual, L_failed);
2679 __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2680 __ jcc(Assembler::greaterEqual, L);
2681 __ stop("must be a primitive array");
2682 __ bind(L);
2699 __ movl(r10_offset, rax_lh);
2700 __ shrl(r10_offset, Klass::_lh_header_size_shift);
2701 __ andptr(r10_offset, Klass::_lh_header_size_mask); // array_offset
2702 __ addptr(src, r10_offset); // src array offset
2703 __ addptr(dst, r10_offset); // dst array offset
2705 __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2715 __ BIND(L_copy_bytes);
2716 __ cmpl(rax_elsize, 0);
2717 __ jccb(Assembler::notEqual, L_copy_shorts);
2718 __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2719 __ lea(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2720 __ movl2ptr(count, r11_length); // length
2721 __ jump(RuntimeAddress(byte_copy_entry));
2723 __ BIND(L_copy_shorts);
2724 __ cmpl(rax_elsize, LogBytesPerShort);
2725 __ jccb(Assembler::notEqual, L_copy_ints);
2726 __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2727 __ lea(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2728 __ movl2ptr(count, r11_length); // length
2729 __ jump(RuntimeAddress(short_copy_entry));
2731 __ BIND(L_copy_ints);
2732 __ cmpl(rax_elsize, LogBytesPerInt);
2733 __ jccb(Assembler::notEqual, L_copy_longs);
2734 __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2735 __ lea(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2736 __ movl2ptr(count, r11_length); // length
2737 __ jump(RuntimeAddress(int_copy_entry));
2739 __ BIND(L_copy_longs);
2744 __ cmpl(rax_elsize, LogBytesPerLong);
2745 __ jcc(Assembler::equal, L);
2746 __ stop("must be long copy, but elsize is wrong");
2747 __ bind(L);
2751 __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2752 __ lea(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2753 __ movl2ptr(count, r11_length); // length
2754 __ jump(RuntimeAddress(long_copy_entry));
2757 __ BIND(L_objArray);
2762 __ load_klass(rax, dst);
2763 __ cmpq(r10_src_klass, rax); // usual case is exact equality
2764 __ jcc(Assembler::notEqual, L_checkcast_copy);
2770 __ lea(from, Address(src, src_pos, TIMES_OOP,
2772 __ lea(to, Address(dst, dst_pos, TIMES_OOP,
2774 __ movl2ptr(count, r11_length); // length
2775 __ BIND(L_plain_copy);
2776 __ jump(RuntimeAddress(oop_copy_entry));
2778 __ BIND(L_checkcast_copy);
2782 __ cmpl(Address(rax, lh_offset), objArray_lh);
2783 __ jcc(Assembler::notEqual, L_failed);
2790 __ load_klass(r11_dst_klass, dst); // reload
2793 __ lea(from, Address(src, src_pos, TIMES_OOP,
2795 __ lea(to, Address(dst, dst_pos, TIMES_OOP,
2797 __ movl(count, length); // length (reloaded)
2805 __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2811 __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2812 __ movl( sco_temp, Address(r11_dst_klass, sco_offset));
2819 __ movptr(r8, r11_dst_klass); // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2820 __ jump(RuntimeAddress(checkcast_copy_entry));
2823 __ BIND(L_failed);
2824 __ xorptr(rax, rax);
2825 __ notptr(rax); // return -1
2826 __ leave(); // required for proper stackwalking of RuntimeStub frame
2827 __ ret(0);
2934 StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2936 __ subq(rsp, 8);
2937 __ movdbl(Address(rsp, 0), xmm0);
2938 __ fld_d(Address(rsp, 0));
2939 __ flog();
2940 __ fstp_d(Address(rsp, 0));
2941 __ movdbl(xmm0, Address(rsp, 0));
2942 __ addq(rsp, 8);
2943 __ ret(0);
2947 StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2949 __ subq(rsp, 8);
2950 __ movdbl(Address(rsp, 0), xmm0);
2951 __ fld_d(Address(rsp, 0));
2952 __ flog10();
2953 __ fstp_d(Address(rsp, 0));
2954 __ movdbl(xmm0, Address(rsp, 0));
2955 __ addq(rsp, 8);
2956 __ ret(0);
2960 StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
2962 __ subq(rsp, 8);
2963 __ movdbl(Address(rsp, 0), xmm0);
2964 __ fld_d(Address(rsp, 0));
2965 __ trigfunc('s');
2966 __ fstp_d(Address(rsp, 0));
2967 __ movdbl(xmm0, Address(rsp, 0));
2968 __ addq(rsp, 8);
2969 __ ret(0);
2973 StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2975 __ subq(rsp, 8);
2976 __ movdbl(Address(rsp, 0), xmm0);
2977 __ fld_d(Address(rsp, 0));
2978 __ trigfunc('c');
2979 __ fstp_d(Address(rsp, 0));
2980 __ movdbl(xmm0, Address(rsp, 0));
2981 __ addq(rsp, 8);
2982 __ ret(0);
2986 StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2988 __ subq(rsp, 8);
2989 __ movdbl(Address(rsp, 0), xmm0);
2990 __ fld_d(Address(rsp, 0));
2991 __ trigfunc('t');
2992 __ fstp_d(Address(rsp, 0));
2993 __ movdbl(xmm0, Address(rsp, 0));
2994 __ addq(rsp, 8);
2995 __ ret(0);
2999 StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
3001 __ subq(rsp, 8);
3002 __ movdbl(Address(rsp, 0), xmm0);
3003 __ fld_d(Address(rsp, 0));
3004 __ exp_with_fallback(0);
3005 __ fstp_d(Address(rsp, 0));
3006 __ movdbl(xmm0, Address(rsp, 0));
3007 __ addq(rsp, 8);
3008 __ ret(0);
3012 StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
3014 __ subq(rsp, 8);
3015 __ movdbl(Address(rsp, 0), xmm1);
3016 __ fld_d(Address(rsp, 0));
3017 __ movdbl(Address(rsp, 0), xmm0);
3018 __ fld_d(Address(rsp, 0));
3019 __ pow_with_fallback(0);
3020 __ fstp_d(Address(rsp, 0));
3021 __ movdbl(xmm0, Address(rsp, 0));
3022 __ addq(rsp, 8);
3023 __ ret(0);
3031 __ align(16);
3033 address start = __ pc();
3034 __ emit_data64( 0x0405060700010203, relocInfo::none );
3035 __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
3042 __ movdqu(xmmdst, Address(key, offset));
3044 __ pshufb(xmmdst, xmm_shuf_mask);
3046 __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3059 __ align(CodeEntryAlignment);
3062 address start = __ pc();
3077 __ enter(); // required for proper stackwalking of RuntimeStub frame
3080 __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3082 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3083 __ movdqu(xmm_result, Address(from, 0)); // get 16 bytes of input
3089 __ pxor(xmm_result, xmm_temp1);
3096 __ aesenc(xmm_result, xmm_temp1);
3097 __ aesenc(xmm_result, xmm_temp2);
3098 __ aesenc(xmm_result, xmm_temp3);
3099 __ aesenc(xmm_result, xmm_temp4);
3106 __ aesenc(xmm_result, xmm_temp1);
3107 __ aesenc(xmm_result, xmm_temp2);
3108 __ aesenc(xmm_result, xmm_temp3);
3109 __ aesenc(xmm_result, xmm_temp4);
3114 __ cmpl(keylen, 44);
3115 __ jccb(Assembler::equal, L_doLast);
3117 __ aesenc(xmm_result, xmm_temp1);
3118 __ aesenc(xmm_result, xmm_temp2);
3123 __ cmpl(keylen, 52);
3124 __ jccb(Assembler::equal, L_doLast);
3126 __ aesenc(xmm_result, xmm_temp1);
3127 __ aesenc(xmm_result, xmm_temp2);
3132 __ BIND(L_doLast);
3133 __ aesenc(xmm_result, xmm_temp1);
3134 __ aesenclast(xmm_result, xmm_temp2);
3135 __ movdqu(Address(to, 0), xmm_result); // store the result
3136 __ xorptr(rax, rax); // return 0
3137 __ leave(); // required for proper stackwalking of RuntimeStub frame
3138 __ ret(0);
3153 __ align(CodeEntryAlignment);
3156 address start = __ pc();
3171 __ enter(); // required for proper stackwalking of RuntimeStub frame
3174 __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3176 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3177 __ movdqu(xmm_result, Address(from, 0));
3187 __ pxor (xmm_result, xmm_temp1);
3188 __ aesdec(xmm_result, xmm_temp2);
3189 __ aesdec(xmm_result, xmm_temp3);
3190 __ aesdec(xmm_result, xmm_temp4);
3197 __ aesdec(xmm_result, xmm_temp1);
3198 __ aesdec(xmm_result, xmm_temp2);
3199 __ aesdec(xmm_result, xmm_temp3);
3200 __ aesdec(xmm_result, xmm_temp4);
3206 __ cmpl(keylen, 44);
3207 __ jccb(Assembler::equal, L_doLast);
3209 __ aesdec(xmm_result, xmm_temp1);
3210 __ aesdec(xmm_result, xmm_temp2);
3215 __ cmpl(keylen, 52);
3216 __ jccb(Assembler::equal, L_doLast);
3218 __ aesdec(xmm_result, xmm_temp1);
3219 __ aesdec(xmm_result, xmm_temp2);
3224 __ BIND(L_doLast);
3225 __ aesdec(xmm_result, xmm_temp1);
3226 __ aesdec(xmm_result, xmm_temp2);
3229 __ aesdeclast(xmm_result, xmm_temp3);
3230 __ movdqu(Address(to, 0), xmm_result); // store the result
3231 __ xorptr(rax, rax); // return 0
3232 __ leave(); // required for proper stackwalking of RuntimeStub frame
3233 __ ret(0);
3250 __ align(CodeEntryAlignment);
3252 address start = __ pc();
3280 __ enter(); // required for proper stackwalking of RuntimeStub frame
3284 __ movl(len_reg, len_mem);
3286 __ subptr(rsp, -rsp_after_call_off * wordSize);
3288 __ movdqu(xmm_save(i), as_XMMRegister(i));
3293 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3299 __ movdqu(xmm_result, Address(rvec, 0x00)); // initialize xmm_result with r vec
3302 __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3303 __ cmpl(rax, 44);
3304 __ jcc(Assembler::notEqual, L_key_192_256);
3307 __ movptr(pos, 0);
3308 __ align(OptoLoopAlignment);
3310 __ BIND(L_loopTop_128);
3311 __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
3312 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
3313 __ pxor (xmm_result, xmm_key0); // do the aes rounds
3315 __ aesenc(xmm_result, as_XMMRegister(rnum));
3317 __ aesenclast(xmm_result, xmm_key10);
3318 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3320 __ addptr(pos, AESBlockSize);
3321 __ subptr(len_reg, AESBlockSize);
3322 __ jcc(Assembler::notEqual, L_loopTop_128);
3324 __ BIND(L_exit);
3325 __ movdqu(Address(rvec, 0), xmm_result); // final value of r stored in rvec of CipherBlockChaining object
3330 __ movdqu(as_XMMRegister(i), xmm_save(i));
3333 __ movl(rax, 0); // return 0 (why?)
3334 __ leave(); // required for proper stackwalking of RuntimeStub frame
3335 __ ret(0);
3337 __ BIND(L_key_192_256);
3341 __ cmpl(rax, 52);
3342 __ jcc(Assembler::notEqual, L_key_256);
3345 __ movptr(pos, 0);
3346 __ align(OptoLoopAlignment);
3348 __ BIND(L_loopTop_192);
3349 __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
3350 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
3351 __ pxor (xmm_result, xmm_key0); // do the aes rounds
3353 __ aesenc(xmm_result, as_XMMRegister(rnum));
3355 __ aesenclast(xmm_result, xmm_key12);
3356 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3358 __ addptr(pos, AESBlockSize);
3359 __ subptr(len_reg, AESBlockSize);
3360 __ jcc(Assembler::notEqual, L_loopTop_192);
3361 __ jmp(L_exit);
3363 __ BIND(L_key_256);
3366 __ movptr(pos, 0);
3367 __ align(OptoLoopAlignment);
3369 __ BIND(L_loopTop_256);
3370 __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
3371 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
3372 __ pxor (xmm_result, xmm_key0); // do the aes rounds
3374 __ aesenc(xmm_result, as_XMMRegister(rnum));
3377 __ aesenclast(xmm_result, xmm_temp);
3378 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3380 __ addptr(pos, AESBlockSize);
3381 __ subptr(len_reg, AESBlockSize);
3382 __ jcc(Assembler::notEqual, L_loopTop_256);
3383 __ jmp(L_exit);
3405 __ align(CodeEntryAlignment);
3407 address start = __ pc();
3431 __ enter(); // required for proper stackwalking of RuntimeStub frame
3435 __ movl(len_reg, len_mem);
3437 __ subptr(rsp, -rsp_after_call_off * wordSize);
3439 __ movdqu(xmm_save(i), as_XMMRegister(i));
3445 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3461 __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
3464 __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3465 __ cmpl(rax, 44);
3466 __ jcc(Assembler::notEqual, L_key_192_256);
3470 __ movptr(pos, 0);
3471 __ align(OptoLoopAlignment);
3472 __ BIND(L_multiBlock_loopTop_128);
3473 __ cmpptr(len_reg, 4*AESBlockSize); // see if at least 4 blocks left
3474 __ jcc(Assembler::less, L_singleBlock_loopTop_128);
3476 __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0*AESBlockSize)); // get next 4 blocks into xmmresult registers
3477 __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1*AESBlockSize));
3478 __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2*AESBlockSize));
3479 __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3*AESBlockSize));
3482 __ opc(xmm_result0, src_reg); \
3483 __ opc(xmm_result1, src_reg); \
3484 __ opc(xmm_result2, src_reg); \
3485 __ opc(xmm_result3, src_reg);
3493 __ pxor(xmm_result0, xmm_prev_block_cipher);
3494 __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0*AESBlockSize));
3495 __ pxor(xmm_result1, xmm_prev_block_cipher);
3496 __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1*AESBlockSize));
3497 __ pxor(xmm_result2, xmm_prev_block_cipher);
3498 __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2*AESBlockSize));
3499 __ pxor(xmm_result3, xmm_prev_block_cipher);
3500 __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3*AESBlockSize)); // this will carry over to next set of blocks
3502 __ movdqu(Address(to, pos, Address::times_1, 0*AESBlockSize), xmm_result0); // store 4 results into the next 64 bytes of output
3503 __ movdqu(Address(to, pos, Address::times_1, 1*AESBlockSize), xmm_result1);
3504 __ movdqu(Address(to, pos, Address::times_1, 2*AESBlockSize), xmm_result2);
3505 __ movdqu(Address(to, pos, Address::times_1, 3*AESBlockSize), xmm_result3);
3507 __ addptr(pos, 4*AESBlockSize);
3508 __ subptr(len_reg, 4*AESBlockSize);
3509 __ jmp(L_multiBlock_loopTop_128);
3519 __ align(OptoLoopAlignment);
3520 __ BIND(L_singleBlock_loopTop_128);
3521 __ cmpptr(len_reg, 0); // any blocks left??
3522 __ jcc(Assembler::equal, L_exit);
3523 __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3524 __ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
3525 __ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
3527 __ aesdec(xmm_result, as_XMMRegister(rnum));
3529 __ aesdeclast(xmm_result, xmm_key_last);
3530 __ pxor (xmm_result, xmm_prev_block_cipher); // xor with the current r vector
3531 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3533 __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
3535 __ addptr(pos, AESBlockSize);
3536 __ subptr(len_reg, AESBlockSize);
3537 __ jmp(L_singleBlock_loopTop_128);
3540 __ BIND(L_exit);
3541 __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
3545 __ movdqu(as_XMMRegister(i), xmm_save(i));
3548 __ movl(rax, 0); // return 0 (why?)
3549 __ leave(); // required for proper stackwalking of RuntimeStub frame
3550 __ ret(0);
3553 __ BIND(L_key_192_256);
3556 __ cmpl(rax, 52);
3557 __ jcc(Assembler::notEqual, L_key_256);
3561 __ movptr(pos, 0);
3562 __ align(OptoLoopAlignment);
3564 __ BIND(L_singleBlock_loopTop_192);
3565 __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3566 __ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
3567 __ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
3569 __ aesdec(xmm_result, as_XMMRegister(rnum));
3571 __ aesdec(xmm_result, xmm_key11);
3572 __ aesdec(xmm_result, xmm_key12);
3573 __ aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0
3574 __ pxor (xmm_result, xmm_prev_block_cipher); // xor with the current r vector
3575 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3577 __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
3578 __ addptr(pos, AESBlockSize);
3579 __ subptr(len_reg, AESBlockSize);
3580 __ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
3581 __ jmp(L_exit);
3583 __ BIND(L_key_256);
3585 __ movptr(pos, 0);
3586 __ align(OptoLoopAlignment);
3588 __ BIND(L_singleBlock_loopTop_256);
3589 __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3590 __ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
3591 __ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
3593 __ aesdec(xmm_result, as_XMMRegister(rnum));
3595 __ aesdec(xmm_result, xmm_key11);
3597 __ aesdec(xmm_result, xmm_temp);
3599 __ aesdec(xmm_result, xmm_temp);
3601 __ aesdec(xmm_result, xmm_temp);
3602 __ aesdeclast(xmm_result, xmm_key_last); // xmm15 came from key+0
3603 __ pxor (xmm_result, xmm_prev_block_cipher); // xor with the current r vector
3604 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3606 __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
3607 __ addptr(pos, AESBlockSize);
3608 __ subptr(len_reg, AESBlockSize);
3609 __ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
3610 __ jmp(L_exit);
3617 #undef __
3618 #define __ masm->
3658 address start = __ pc();
3665 __ enter(); // required for proper stackwalking of RuntimeStub frame
3670 __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
3672 int frame_complete = __ pc() - start;
3675 address the_pc = __ pc();
3676 __ set_last_Java_frame(rsp, rbp, the_pc);
3677 __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
3682 __ movptr(c_rarg1, arg1);
3685 __ movptr(c_rarg2, arg2);
3687 __ movptr(c_rarg0, r15_thread);
3689 __ call(RuntimeAddress(runtime_entry));
3696 __ reset_last_Java_frame(true, true);
3698 __ leave(); // required for proper stackwalking of RuntimeStub frame
3703 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
3705 __ jcc(Assembler::notEqual, L);
3706 __ should_not_reach_here();
3707 __ bind(L);
3709 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));