Lines Matching defs:__
61 #define __ _masm->
67 #define BLOCK_COMMENT(str) __ block_comment(str)
100 __ incrementl(ExternalAddress((address)&counter));
148 address start = __ pc();
169 __ enter();
170 __ movptr(rcx, parameter_size); // parameter counter
171 __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
172 __ addptr(rcx, locals_count_in_bytes); // reserve space for register saves
173 __ subptr(rsp, rcx);
174 __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
177 __ movptr(saved_rdi, rdi);
178 __ movptr(saved_rsi, rsi);
179 __ movptr(saved_rbx, rbx);
183 __ stmxcsr(mxcsr_save);
184 __ movl(rax, mxcsr_save);
185 __ andl(rax, MXCSR_MASK); // Only check control and mask bits
187 __ cmp32(rax, mxcsr_std);
188 __ jcc(Assembler::equal, skip_ldmx);
189 __ ldmxcsr(mxcsr_std);
190 __ bind(skip_ldmx);
194 __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
199 __ movptr(rcx, thread);
200 __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
201 __ jcc(Assembler::equal, L);
202 __ stop("StubRoutines::call_stub: entered with pending exception");
203 __ bind(L);
210 __ movl(rcx, parameter_size); // parameter counter
211 __ testl(rcx, rcx);
212 __ jcc(Assembler::zero, parameters_done);
222 __ movptr(rdx, parameters); // parameter pointer
223 __ xorptr(rbx, rbx);
225 __ BIND(loop);
228 __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
229 __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
231 __ increment(rbx);
232 __ decrement(rcx);
233 __ jcc(Assembler::notZero, loop);
236 __ BIND(parameters_done);
237 __ movptr(rbx, method); // get methodOop
238 __ movptr(rax, entry_point); // get entry_point
239 __ mov(rsi, rsp); // set sender sp
241 __ call(rax);
244 return_address = __ pc();
250 __ verify_FPU(0, "call_stub_return");
253 __ ffree(i);
257 __ movl(rsi, result_type);
258 __ cmpl(rsi, T_DOUBLE);
259 __ jcc(Assembler::equal, L_skip);
262 __ cmpl(rsi, T_FLOAT);
263 __ jcc(Assembler::equal, L_skip);
265 __ ffree(0);
267 __ BIND(L_skip);
273 __ movptr(rdi, result);
275 __ movl(rsi, result_type);
276 __ cmpl(rsi, T_LONG);
277 __ jcc(Assembler::equal, is_long);
278 __ cmpl(rsi, T_FLOAT);
279 __ jcc(Assembler::equal, is_float);
280 __ cmpl(rsi, T_DOUBLE);
281 __ jcc(Assembler::equal, is_double);
284 __ movl(Address(rdi, 0), rax);
285 __ BIND(exit);
288 __ verify_FPU(0, "generate_call_stub");
291 __ lea(rsp, rsp_after_call);
295 __ ldmxcsr(mxcsr_save);
299 __ movptr(rbx, saved_rbx);
300 __ movptr(rsi, saved_rsi);
301 __ movptr(rdi, saved_rdi);
302 __ addptr(rsp, 4*wordSize);
305 __ pop(rbp);
306 __ ret(0);
309 __ BIND(is_long);
310 __ movl(Address(rdi, 0 * wordSize), rax);
311 __ movl(Address(rdi, 1 * wordSize), rdx);
312 __ jmp(exit);
314 __ BIND(is_float);
317 __ movflt(Address(rdi, 0), xmm0);
319 __ fstp_s(Address(rdi, 0));
321 __ jmp(exit);
323 __ BIND(is_double);
326 __ movdbl(Address(rdi, 0), xmm0);
328 __ fstp_d(Address(rdi, 0));
330 __ jmp(exit);
351 address start = __ pc();
354 __ movptr(rcx, thread);
358 __ get_thread(rbx);
359 __ cmpptr(rbx, rcx);
360 __ jcc(Assembler::equal, L);
361 __ stop("StubRoutines::catch_exception: threads must correspond");
362 __ bind(L);
366 __ verify_oop(rax);
367 __ movptr(Address(rcx, Thread::pending_exception_offset()), rax );
368 __ lea(Address(rcx, Thread::exception_file_offset ()),
370 __ movl(Address(rcx, Thread::exception_line_offset ()), __LINE__ );
373 __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
392 address start = __ pc();
411 __ get_thread(thread);
412 __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
413 __ jcc(Assembler::notEqual, L);
414 __ stop("StubRoutines::forward exception: no pending exception (1)");
415 __ bind(L);
420 __ get_thread(thread);
421 __ movptr(exception_pc, Address(rsp, 0));
423 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
424 __ mov(handler_addr, rax);
427 __ get_thread(thread);
428 __ pop(exception_pc);
429 __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
430 __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
435 __ testptr(exception_oop, exception_oop);
436 __ jcc(Assembler::notEqual, L);
437 __ stop("StubRoutines::forward exception: no pending exception (2)");
438 __ bind(L);
443 __ verify_oop(exception_oop);
449 __ jmp(handler_addr);
471 address start = __ pc();
473 __ push(rdx);
476 __ movl(rax, exchange);
477 __ movptr(rdx, dest_addr);
478 __ xchgl(rax, Address(rdx, 0));
479 __ pop(rdx);
480 __ ret(0);
495 address start = __ pc();
502 __ push(rax);
503 __ subptr(rsp, wordSize); // allocate a temp location
504 __ stmxcsr(mxcsr_save);
505 __ movl(rax, mxcsr_save);
506 __ andl(rax, MXCSR_MASK);
507 __ cmp32(rax, mxcsr_std);
508 __ jcc(Assembler::equal, ok_ret);
510 __ warn("MXCSR changed by native JNI code.");
512 __ ldmxcsr(mxcsr_std);
514 __ bind(ok_ret);
515 __ addptr(rsp, wordSize);
516 __ pop(rax);
519 __ ret(0);
534 address start = __ pc();
540 __ push(rax);
541 __ subptr(rsp, wordSize); // allocate a temp location
542 __ fnstcw(fpu_cntrl_wrd_save);
543 __ movl(rax, fpu_cntrl_wrd_save);
544 __ andl(rax, FPU_CNTRL_WRD_MASK);
546 __ cmp32(rax, fpu_std);
547 __ jcc(Assembler::equal, ok_ret);
549 __ warn("Floating point control word changed by native JNI code.");
551 __ fldcw(fpu_std);
553 __ bind(ok_ret);
554 __ addptr(rsp, wordSize);
555 __ pop(rax);
558 __ ret(0);
572 address start = __ pc();
589 __ subptr(rsp, wordSize * 2);
590 __ fstp_d(Address(rsp, 0));
593 __ push(rbx);
594 __ push(rcx);
595 __ push(rsi);
596 __ push(rdi);
597 __ push(rbp);
598 __ push_FPU_state();
602 __ fld_d(Address(rsp, saved_argument_off * wordSize));
604 __ subptr(rsp, wordSize*2);
605 __ fst_d(Address(rsp, 0));
608 __ empty_FPU_stack();
614 __ call_VM_leaf( fcn, 2 );
617 __ pop_FPU_state();
618 __ pop(rbp);
619 __ pop(rdi);
620 __ pop(rsi);
621 __ pop(rcx);
622 __ pop(rbx);
623 __ addptr(rsp, wordSize * 2);
625 __ ret(0);
637 address start = __ pc();
639 __ push(0); // hole for return address-to-be
640 __ pusha(); // push registers
643 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
644 __ movptr(next_pc, rax); // stuff next address
645 __ popa();
646 __ ret(0); // jump to next address
657 address start = __ pc();
669 __ pushf();
670 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
671 __ push(rdx); // save rdx
673 __ movptr(rax, Address(rsp, 4 * wordSize)); // get object
674 __ testptr(rax, rax);
675 __ jcc(Assembler::zero, exit); // if obj is NULL it is ok
680 __ mov(rdx, rax);
681 __ andptr(rdx, oop_mask);
682 __ cmpptr(rdx, oop_bits);
683 __ jcc(Assembler::notZero, error);
686 __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
687 __ testptr(rax, rax);
688 __ jcc(Assembler::zero, error); // if klass is NULL it is broken
693 __ mov(rdx, rax);
694 __ andptr(rdx, klass_mask);
695 __ cmpptr(rdx, klass_bits);
696 __ jcc(Assembler::notZero, error);
699 __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass
700 __ testptr(rax, rax);
701 __ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
703 __ mov(rdx, rax);
704 __ andptr(rdx, klass_mask);
705 __ cmpptr(rdx, klass_bits);
706 __ jcc(Assembler::notZero, error); // if klass not in right area
710 __ bind(exit);
711 __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
712 __ pop(rdx); // restore rdx
713 __ popf(); // restore EFLAGS
714 __ ret(3 * wordSize); // pop arguments
717 __ bind(error);
718 __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
719 __ pop(rdx); // get saved rdx back
720 __ popf(); // get saved EFLAGS off stack -- will be ignored
721 __ pusha(); // push registers (eip = return address & msg are already pushed)
723 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
724 __ popa();
725 __ ret(3 * wordSize); // pop arguments
743 __ pusha(); // push registers
744 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
746 __ popa();
775 __ pusha(); // push registers
776 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
778 __ popa();
792 __ lea(end, Address(start, count, Address::times_ptr, -wordSize));
793 __ shrptr(start, CardTableModRefBS::card_shift);
794 __ shrptr(end, CardTableModRefBS::card_shift);
795 __ subptr(end, start); // end --> count
796 __ BIND(L_loop);
799 __ movb(cardtable, 0);
800 __ decrement(count);
801 __ jcc(Assembler::greaterEqual, L_loop);
824 __ jmpb(L_copy_64_bytes);
825 __ align(OptoLoopAlignment);
826 __ BIND(L_copy_64_bytes_loop);
830 __ vmovdqu(xmm0, Address(from, 0));
831 __ vmovdqu(Address(from, to_from, Address::times_1, 0), xmm0);
832 __ vmovdqu(xmm1, Address(from, 32));
833 __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
835 __ movdqu(xmm0, Address(from, 0));
836 __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
837 __ movdqu(xmm1, Address(from, 16));
838 __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
839 __ movdqu(xmm2, Address(from, 32));
840 __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
841 __ movdqu(xmm3, Address(from, 48));
842 __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
845 __ movq(xmm0, Address(from, 0));
846 __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
847 __ movq(xmm1, Address(from, 8));
848 __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
849 __ movq(xmm2, Address(from, 16));
850 __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
851 __ movq(xmm3, Address(from, 24));
852 __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
853 __ movq(xmm4, Address(from, 32));
854 __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
855 __ movq(xmm5, Address(from, 40));
856 __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
857 __ movq(xmm6, Address(from, 48));
858 __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
859 __ movq(xmm7, Address(from, 56));
860 __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
863 __ addl(from, 64);
864 __ BIND(L_copy_64_bytes);
865 __ subl(qword_count, 8);
866 __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
870 __ vzeroupper();
872 __ addl(qword_count, 8);
873 __ jccb(Assembler::zero, L_exit);
877 __ BIND(L_copy_8_bytes);
878 __ movq(xmm0, Address(from, 0));
879 __ movq(Address(from, to_from, Address::times_1), xmm0);
880 __ addl(from, 8);
881 __ decrement(qword_count);
882 __ jcc(Assembler::greater, L_copy_8_bytes);
883 __ BIND(L_exit);
897 __ jmpb(L_copy_64_bytes);
898 __ align(OptoLoopAlignment);
899 __ BIND(L_copy_64_bytes_loop);
900 __ movq(mmx0, Address(from, 0));
901 __ movq(mmx1, Address(from, 8));
902 __ movq(mmx2, Address(from, 16));
903 __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
904 __ movq(mmx3, Address(from, 24));
905 __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
906 __ movq(mmx4, Address(from, 32));
907 __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
908 __ movq(mmx5, Address(from, 40));
909 __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
910 __ movq(mmx6, Address(from, 48));
911 __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
912 __ movq(mmx7, Address(from, 56));
913 __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
914 __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
915 __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
916 __ addptr(from, 64);
917 __ BIND(L_copy_64_bytes);
918 __ subl(qword_count, 8);
919 __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
920 __ addl(qword_count, 8);
921 __ jccb(Assembler::zero, L_exit);
925 __ BIND(L_copy_8_bytes);
926 __ movq(mmx0, Address(from, 0));
927 __ movq(Address(from, to_from, Address::times_1), mmx0);
928 __ addptr(from, 8);
929 __ decrement(qword_count);
930 __ jcc(Assembler::greater, L_copy_8_bytes);
931 __ BIND(L_exit);
932 __ emms();
939 __ align(CodeEntryAlignment);
941 address start = __ pc();
954 __ enter(); // required for proper stackwalking of RuntimeStub frame
955 __ push(rsi);
956 __ push(rdi);
957 __ movptr(from , Address(rsp, 12+ 4));
958 __ movptr(to , Address(rsp, 12+ 8));
959 __ movl(count, Address(rsp, 12+ 12));
962 *entry = __ pc(); // Entry point from conjoint arraycopy stub.
967 __ testl(count, count);
968 __ jcc(Assembler::zero, L_0_count);
970 __ mov(saved_to, to); // save 'to'
973 __ subptr(to, from); // to --> to_from
974 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
975 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
980 __ testl(from, 1);
981 __ jccb(Assembler::zero, L_skip_align1);
982 __ movb(rax, Address(from, 0));
983 __ movb(Address(from, to_from, Address::times_1, 0), rax);
984 __ increment(from);
985 __ decrement(count);
986 __ BIND(L_skip_align1);
989 __ testl(from, 2);
990 __ jccb(Assembler::zero, L_skip_align2);
991 __ movw(rax, Address(from, 0));
992 __ movw(Address(from, to_from, Address::times_1, 0), rax);
993 __ addptr(from, 2);
994 __ subl(count, 1<<(shift-1));
995 __ BIND(L_skip_align2);
998 __ mov(rax, count); // save 'count'
999 __ shrl(count, shift); // bytes count
1000 __ addptr(to_from, from);// restore 'to'
1001 __ rep_mov();
1002 __ subptr(to_from, from);// restore 'to_from'
1003 __ mov(count, rax); // restore 'count'
1004 __ jmpb(L_copy_2_bytes); // all dwords were copied
1008 __ testptr(from, 4);
1009 __ jccb(Assembler::zero, L_copy_64_bytes);
1010 __ movl(rax, Address(from, 0));
1011 __ movl(Address(from, to_from, Address::times_1, 0), rax);
1012 __ addptr(from, 4);
1013 __ subl(count, 1<<shift);
1015 __ BIND(L_copy_64_bytes);
1016 __ mov(rax, count);
1017 __ shrl(rax, shift+1); // 8 bytes chunk count
1028 __ BIND(L_copy_4_bytes);
1029 __ testl(count, 1<<shift);
1030 __ jccb(Assembler::zero, L_copy_2_bytes);
1031 __ movl(rax, Address(from, 0));
1032 __ movl(Address(from, to_from, Address::times_1, 0), rax);
1034 __ addptr(from, 4);
1035 __ BIND(L_copy_2_bytes);
1037 __ testl(count, 1<<(shift-1));
1038 __ jccb(Assembler::zero, L_copy_byte);
1039 __ movw(rax, Address(from, 0));
1040 __ movw(Address(from, to_from, Address::times_1, 0), rax);
1042 __ addptr(from, 2);
1043 __ BIND(L_copy_byte);
1045 __ testl(count, 1);
1046 __ jccb(Assembler::zero, L_exit);
1047 __ movb(rax, Address(from, 0));
1048 __ movb(Address(from, to_from, Address::times_1, 0), rax);
1049 __ BIND(L_exit);
1051 __ BIND(L_copy_byte);
1054 __ BIND(L_copy_2_bytes);
1058 __ movl(count, Address(rsp, 12+12)); // reread 'count'
1059 __ mov(to, saved_to); // restore 'to'
1061 __ BIND(L_0_count);
1064 __ pop(rdi);
1065 __ pop(rsi);
1066 __ leave(); // required for proper stackwalking of RuntimeStub frame
1067 __ xorptr(rax, rax); // return 0
1068 __ ret(0);
1074 __ align(CodeEntryAlignment);
1076 address start = __ pc();
1084 __ enter(); // required for proper stackwalking of RuntimeStub frame
1085 __ push(rsi);
1086 __ push(rdi);
1087 __ movptr(to , Address(rsp, 12+ 4));
1088 __ movl(value, Address(rsp, 12+ 8));
1089 __ movl(count, Address(rsp, 12+ 12));
1091 __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1093 __ pop(rdi);
1094 __ pop(rsi);
1095 __ leave(); // required for proper stackwalking of RuntimeStub frame
1096 __ ret(0);
1105 __ align(CodeEntryAlignment);
1107 address start = __ pc();
1121 __ enter(); // required for proper stackwalking of RuntimeStub frame
1122 __ push(rsi);
1123 __ push(rdi);
1124 __ movptr(src , Address(rsp, 12+ 4)); // from
1125 __ movptr(dst , Address(rsp, 12+ 8)); // to
1126 __ movl2ptr(count, Address(rsp, 12+12)); // count
1129 *entry = __ pc(); // Entry point from generic arraycopy stub.
1134 __ mov(from, src);
1135 __ mov(to , dst);
1139 __ cmpptr(dst, src);
1140 __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1141 __ jump_cc(Assembler::belowEqual, nooverlap);
1142 __ cmpptr(dst, end);
1143 __ jump_cc(Assembler::aboveEqual, nooverlap);
1146 __ testl(count, count);
1147 __ jcc(Assembler::zero, L_0_count);
1152 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1153 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1156 __ lea(end, Address(dst, count, sf, 0));
1159 __ testl(end, 1);
1160 __ jccb(Assembler::zero, L_skip_align1);
1161 __ decrement(count);
1162 __ movb(rdx, Address(from, count, sf, 0));
1163 __ movb(Address(to, count, sf, 0), rdx);
1164 __ BIND(L_skip_align1);
1167 __ testl(end, 2);
1168 __ jccb(Assembler::zero, L_skip_align2);
1169 __ subptr(count, 1<<(shift-1));
1170 __ movw(rdx, Address(from, count, sf, 0));
1171 __ movw(Address(to, count, sf, 0), rdx);
1172 __ BIND(L_skip_align2);
1173 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1174 __ jcc(Assembler::below, L_copy_4_bytes);
1178 __ std();
1179 __ mov(rax, count); // Save 'count'
1180 __ mov(rdx, to); // Save 'to'
1181 __ lea(rsi, Address(from, count, sf, -4));
1182 __ lea(rdi, Address(to , count, sf, -4));
1183 __ shrptr(count, shift); // bytes count
1184 __ rep_mov();
1185 __ cld();
1186 __ mov(count, rax); // restore 'count'
1187 __ andl(count, (1<<shift)-1); // mask the number of rest elements
1188 __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1189 __ mov(to, rdx); // restore 'to'
1190 __ jmpb(L_copy_2_bytes); // all dword were copied
1193 __ testptr(end, 4);
1194 __ jccb(Assembler::zero, L_copy_8_bytes);
1195 __ subl(count, 1<<shift);
1196 __ movl(rdx, Address(from, count, sf, 0));
1197 __ movl(Address(to, count, sf, 0), rdx);
1198 __ jmpb(L_copy_8_bytes);
1200 __ align(OptoLoopAlignment);
1202 __ BIND(L_copy_8_bytes_loop);
1204 __ movq(xmm0, Address(from, count, sf, 0));
1205 __ movq(Address(to, count, sf, 0), xmm0);
1207 __ movq(mmx0, Address(from, count, sf, 0));
1208 __ movq(Address(to, count, sf, 0), mmx0);
1210 __ BIND(L_copy_8_bytes);
1211 __ subl(count, 2<<shift);
1212 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1213 __ addl(count, 2<<shift);
1215 __ emms();
1218 __ BIND(L_copy_4_bytes);
1220 __ testl(count, 1<<shift);
1221 __ jccb(Assembler::zero, L_copy_2_bytes);
1222 __ movl(rdx, Address(from, count, sf, -4));
1223 __ movl(Address(to, count, sf, -4), rdx);
1226 __ subl(count, (1<<shift));
1227 __ BIND(L_copy_2_bytes);
1229 __ testl(count, 1<<(shift-1));
1230 __ jccb(Assembler::zero, L_copy_byte);
1231 __ movw(rdx, Address(from, count, sf, -2));
1232 __ movw(Address(to, count, sf, -2), rdx);
1234 __ subl(count, 1<<(shift-1));
1235 __ BIND(L_copy_byte);
1237 __ testl(count, 1);
1238 __ jccb(Assembler::zero, L_exit);
1239 __ movb(rdx, Address(from, 0));
1240 __ movb(Address(to, 0), rdx);
1241 __ BIND(L_exit);
1243 __ BIND(L_copy_byte);
1246 __ BIND(L_copy_2_bytes);
1249 __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1251 __ BIND(L_0_count);
1254 __ pop(rdi);
1255 __ pop(rsi);
1256 __ leave(); // required for proper stackwalking of RuntimeStub frame
1257 __ xorptr(rax, rax); // return 0
1258 __ ret(0);
1264 __ align(CodeEntryAlignment);
1266 address start = __ pc();
1274 __ enter(); // required for proper stackwalking of RuntimeStub frame
1275 __ movptr(from , Address(rsp, 8+0)); // from
1276 __ movptr(to , Address(rsp, 8+4)); // to
1277 __ movl2ptr(count, Address(rsp, 8+8)); // count
1279 *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1282 __ subptr(to, from); // to --> to_from
1290 __ jmpb(L_copy_8_bytes);
1291 __ align(OptoLoopAlignment);
1292 __ BIND(L_copy_8_bytes_loop);
1293 __ fild_d(Address(from, 0));
1294 __ fistp_d(Address(from, to_from, Address::times_1));
1295 __ addptr(from, 8);
1296 __ BIND(L_copy_8_bytes);
1297 __ decrement(count);
1298 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1301 __ leave(); // required for proper stackwalking of RuntimeStub frame
1302 __ xorptr(rax, rax); // return 0
1303 __ ret(0);
1309 __ align(CodeEntryAlignment);
1311 address start = __ pc();
1319 __ enter(); // required for proper stackwalking of RuntimeStub frame
1320 __ movptr(from , Address(rsp, 8+0)); // from
1321 __ movptr(to , Address(rsp, 8+4)); // to
1322 __ movl2ptr(count, Address(rsp, 8+8)); // count
1324 *entry = __ pc(); // Entry point from generic arraycopy stub.
1328 __ cmpptr(to, from);
1330 __ jump_cc(Assembler::belowEqual, nooverlap);
1331 __ lea(end_from, Address(from, count, Address::times_8, 0));
1332 __ cmpptr(to, end_from);
1333 __ movptr(from, Address(rsp, 8)); // from
1334 __ jump_cc(Assembler::aboveEqual, nooverlap);
1336 __ jmpb(L_copy_8_bytes);
1338 __ align(OptoLoopAlignment);
1339 __ BIND(L_copy_8_bytes_loop);
1342 __ movq(xmm0, Address(from, count, Address::times_8));
1343 __ movq(Address(to, count, Address::times_8), xmm0);
1345 __ movq(mmx0, Address(from, count, Address::times_8));
1346 __ movq(Address(to, count, Address::times_8), mmx0);
1349 __ fild_d(Address(from, count, Address::times_8));
1350 __ fistp_d(Address(to, count, Address::times_8));
1352 __ BIND(L_copy_8_bytes);
1353 __ decrement(count);
1354 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1357 __ emms();
1360 __ leave(); // required for proper stackwalking of RuntimeStub frame
1361 __ xorptr(rax, rax); // return 0
1362 __ ret(0);
1379 if (label_ptr != NULL) __ jcc(assembler_con, *(label_ptr)); \
1380 else __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1384 // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1391 __ cmpptr(sub_klass, super_klass_addr);
1395 __ movl2ptr(temp, super_check_offset_addr);
1397 __ movptr(temp, super_check_addr); // load displayed supertype
1398 __ cmpptr(temp, super_klass_addr); // test the super type
1402 __ cmpl(super_check_offset_addr, sc_offset);
1408 __ movptr(super_klass, super_klass_addr);
1409 __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1412 __ bind(L_fallthrough);
1435 __ align(CodeEntryAlignment);
1437 address start = __ pc();
1452 __ enter(); // required for proper stackwalking of RuntimeStub frame
1454 __ push(rsi);
1455 __ push(rdi);
1456 __ push(rbx);
1465 __ movptr(from, from_arg);
1466 __ movptr(to, to_arg);
1467 __ movl2ptr(length, length_arg);
1470 *entry = __ pc(); // Entry point from generic arraycopy stub.
1496 __ lea(end_from, end_from_addr);
1497 __ lea(end_to, end_to_addr);
1499 __ negptr(count); // negate and test the length
1500 __ jccb(Assembler::notZero, L_load_element);
1503 __ xorptr(rax, rax); // return 0 on (trivial) success
1504 __ jmp(L_done);
1511 __ align(OptoLoopAlignment);
1513 __ BIND(L_store_element);
1514 __ movptr(to_element_addr, elem); // store the oop
1515 __ increment(count); // increment the count toward zero
1516 __ jccb(Assembler::zero, L_do_card_marks);
1519 __ BIND(L_load_element);
1520 __ movptr(elem, from_element_addr); // load the oop
1521 __ testptr(elem, elem);
1522 __ jccb(Assembler::zero, L_store_element);
1527 __ movptr(elem_klass, elem_klass_addr); // query the object klass
1539 __ addl(count, length_arg); // transfers = (length - remaining)
1540 __ movl2ptr(rax, count); // save the value
1541 __ notptr(rax); // report (-1^K) to caller (does not affect flags)
1542 __ jccb(Assembler::notZero, L_post_barrier);
1543 __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1546 __ BIND(L_do_card_marks);
1547 __ xorptr(rax, rax); // return 0 on success
1548 __ movl2ptr(count, length_arg);
1550 __ BIND(L_post_barrier);
1551 __ movptr(to, to_arg); // reload
1555 __ BIND(L_done);
1556 __ pop(rbx);
1557 __ pop(rdi);
1558 __ pop(rsi);
1560 __ leave(); // required for proper stackwalking of RuntimeStub frame
1561 __ ret(0);
1591 __ align(CodeEntryAlignment);
1593 address start = __ pc();
1599 __ enter(); // required for proper stackwalking of RuntimeStub frame
1600 __ push(rsi);
1601 __ push(rdi);
1607 __ movptr(from , from_arg);
1608 __ movptr(to , to_arg);
1609 __ movl2ptr(count, count_arg);
1615 __ mov(bits, from);
1616 __ orptr(bits, to);
1617 __ orptr(bits, count);
1619 __ testl(bits, BytesPerLong-1);
1620 __ jccb(Assembler::zero, L_long_aligned);
1622 __ testl(bits, BytesPerInt-1);
1623 __ jccb(Assembler::zero, L_int_aligned);
1625 __ testl(bits, BytesPerShort-1);
1626 __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1628 __ BIND(L_short_aligned);
1629 __ shrptr(count, LogBytesPerShort); // size => short_count
1630 __ movl(count_arg, count); // update 'count'
1631 __ jump(RuntimeAddress(short_copy_entry));
1633 __ BIND(L_int_aligned);
1634 __ shrptr(count, LogBytesPerInt); // size => int_count
1635 __ movl(count_arg, count); // update 'count'
1636 __ jump(RuntimeAddress(int_copy_entry));
1638 __ BIND(L_long_aligned);
1639 __ shrptr(count, LogBytesPerLong); // size => qword_count
1640 __ movl(count_arg, count); // update 'count'
1641 __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1642 __ pop(rsi);
1643 __ jump(RuntimeAddress(long_copy_entry));
1660 __ addl(src_end, length); // src_pos + length
1661 __ addl(dst_end, length); // dst_pos + length
1664 __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1665 __ jcc(Assembler::above, L_failed);
1668 __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1669 __ jcc(Assembler::above, L_failed);
1700 int advance = target - (__ offset() % modulus);
1702 if (advance > 0) __ nop(advance);
1707 __ BIND(L_failed_0);
1708 __ jmp(L_failed);
1709 assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1711 __ align(CodeEntryAlignment);
1712 address start = __ pc();
1714 __ enter(); // required for proper stackwalking of RuntimeStub frame
1715 __ push(rsi);
1716 __ push(rdi);
1749 __ movptr(src, SRC); // src oop
1750 __ testptr(src, src);
1751 __ jccb(Assembler::zero, L_failed_0);
1754 __ movl2ptr(src_pos, SRC_POS); // src_pos
1755 __ testl(src_pos, src_pos);
1756 __ jccb(Assembler::negative, L_failed_0);
1759 __ movptr(dst, DST); // dst oop
1760 __ testptr(dst, dst);
1761 __ jccb(Assembler::zero, L_failed_0);
1764 __ movl2ptr(dst_pos, DST_POS); // dst_pos
1765 __ testl(dst_pos, dst_pos);
1766 __ jccb(Assembler::negative, L_failed_0);
1769 __ movl2ptr(length, LENGTH); // length
1770 __ testl(length, length);
1771 __ jccb(Assembler::negative, L_failed_0);
1777 __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1783 __ testptr(rcx_src_klass, rcx_src_klass);
1784 __ jccb(Assembler::notZero, L2); // it is broken if klass is NULL
1785 __ bind(L1);
1786 __ stop("broken null klass");
1787 __ bind(L2);
1788 __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1789 __ jccb(Assembler::equal, L1); // this would be broken also
1807 __ cmpl(src_klass_lh_addr, objArray_lh);
1808 __ jcc(Assembler::equal, L_objArray);
1811 __ cmpptr(rcx_src_klass, dst_klass_addr);
1812 __ jccb(Assembler::notEqual, L_failed_0);
1816 __ movl(rcx_lh, src_klass_lh_addr);
1819 __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1820 __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1825 __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1826 __ jcc(Assembler::greaterEqual, L); // signed cmp
1827 __ stop("must be a primitive array");
1828 __ bind(L);
1845 __ mov(rsi_offset, rcx_lh);
1846 __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1847 __ andptr(rsi_offset, Klass::_lh_header_size_mask); // array_offset
1848 __ addptr(src_array, rsi_offset); // src array offset
1849 __ addptr(dst_array, rsi_offset); // dst array offset
1850 __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1862 __ movl2ptr(rsi, SRC_POS); // src_pos
1863 __ shlptr(rsi); // src_pos << rcx (log2 elsize)
1865 __ addptr(from, rsi); // from = src_array + SRC_POS << log2 elsize
1866 __ movl2ptr(rdi, DST_POS); // dst_pos
1867 __ shlptr(rdi); // dst_pos << rcx (log2 elsize)
1869 __ addptr(to, rdi); // to = dst_array + DST_POS << log2 elsize
1870 __ movptr(FROM, from); // src_addr
1871 __ mov(rdi_elsize, rcx_lh); // log2 elsize
1872 __ movl2ptr(count, LENGTH); // elements count
1875 __ cmpl(rdi_elsize, 0);
1877 __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1878 __ cmpl(rdi_elsize, LogBytesPerShort);
1879 __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1880 __ cmpl(rdi_elsize, LogBytesPerInt);
1881 __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1883 __ cmpl(rdi_elsize, LogBytesPerLong);
1884 __ jccb(Assembler::notEqual, L_failed);
1886 __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1887 __ pop(rsi);
1888 __ jump(RuntimeAddress(entry_jlong_arraycopy));
1890 __ BIND(L_failed);
1891 __ xorptr(rax, rax);
1892 __ notptr(rax); // return -1
1893 __ pop(rdi);
1894 __ pop(rsi);
1895 __ leave(); // required for proper stackwalking of RuntimeStub frame
1896 __ ret(0);
1899 __ BIND(L_objArray);
1904 __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1905 __ jccb(Assembler::notEqual, L_checkcast_copy);
1911 __ BIND(L_plain_copy);
1912 __ movl2ptr(count, LENGTH); // elements count
1913 __ movl2ptr(src_pos, SRC_POS); // reload src_pos
1914 __ lea(from, Address(src, src_pos, Address::times_ptr,
1916 __ movl2ptr(dst_pos, DST_POS); // reload dst_pos
1917 __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1919 __ movptr(FROM, from); // src_addr
1920 __ movptr(TO, to); // dst_addr
1921 __ movl(COUNT, count); // count
1922 __ jump(RuntimeAddress(entry_oop_arraycopy));
1924 __ BIND(L_checkcast_copy);
1938 __ movptr(rsi_dst_klass, dst_klass_addr);
1939 __ cmpl(dst_klass_lh_addr, objArray_lh);
1940 __ jccb(Assembler::notEqual, L_failed);
1943 __ movl2ptr(src_pos, SRC_POS); // reload rsi
1948 __ push(rbx);
1951 __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1952 __ movptr(rsi_dst_klass, dst_klass_addr);
1959 __ pop(rbx);
1960 __ jmp(L_plain_copy);
1962 __ BIND(L_fail_array_check);
1983 __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1984 __ movl2ptr(length, LENGTH_arg); // reload elements count
1985 __ movl2ptr(src_pos, SRC_POS_arg); // reload src_pos
1986 __ movl2ptr(dst_pos, DST_POS_arg); // reload dst_pos
1988 __ movptr(ckval_arg, rbx); // destination element type
1989 __ movl(rbx, Address(rbx, sco_offset));
1990 __ movl(ckoff_arg, rbx); // corresponding class check offset
1992 __ movl(length_arg, length); // outgoing length argument
1994 __ lea(from, Address(src, src_pos, Address::times_ptr,
1996 __ movptr(from_arg, from);
1998 __ lea(to, Address(dst, dst_pos, Address::times_ptr,
2000 __ movptr(to_arg, to);
2001 __ jump(RuntimeAddress(entry_checkcast_arraycopy));
2114 StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2116 __ fld_d(Address(rsp, 4));
2117 __ flog();
2118 __ ret(0);
2122 StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2124 __ fld_d(Address(rsp, 4));
2125 __ flog10();
2126 __ ret(0);
2130 StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
2132 __ fld_d(Address(rsp, 4));
2133 __ trigfunc('s');
2134 __ ret(0);
2138 StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2140 __ fld_d(Address(rsp, 4));
2141 __ trigfunc('c');
2142 __ ret(0);
2146 StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2148 __ fld_d(Address(rsp, 4));
2149 __ trigfunc('t');
2150 __ ret(0);
2154 StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
2156 __ fld_d(Address(rsp, 4));
2157 __ exp_with_fallback(0);
2158 __ ret(0);
2162 StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
2164 __ fld_d(Address(rsp, 12));
2165 __ fld_d(Address(rsp, 4));
2166 __ pow_with_fallback(0);
2167 __ ret(0);
2175 __ align(16);
2177 address start = __ pc();
2178 __ emit_data(0x00010203, relocInfo::none, 0 );
2179 __ emit_data(0x04050607, relocInfo::none, 0 );
2180 __ emit_data(0x08090a0b, relocInfo::none, 0 );
2181 __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2188 __ movdqu(xmmdst, Address(key, offset));
2190 __ pshufb(xmmdst, xmm_shuf_mask);
2192 __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2200 __ aesenc(xmmdst, xmmtmp);
2207 __ aesdec(xmmdst, xmmtmp);
2220 __ align(CodeEntryAlignment);
2223 address start = __ pc();
2240 __ enter(); // required for proper stackwalking of RuntimeStub frame
2241 __ movptr(from, from_param);
2242 __ movptr(key, key_param);
2245 __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2247 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2248 __ movdqu(xmm_result, Address(from, 0)); // get 16 bytes of input
2249 __ movptr(to, to_param);
2254 __ pxor(xmm_result, xmm_temp1);
2261 __ aesenc(xmm_result, xmm_temp1);
2262 __ aesenc(xmm_result, xmm_temp2);
2263 __ aesenc(xmm_result, xmm_temp3);
2264 __ aesenc(xmm_result, xmm_temp4);
2271 __ aesenc(xmm_result, xmm_temp1);
2272 __ aesenc(xmm_result, xmm_temp2);
2273 __ aesenc(xmm_result, xmm_temp3);
2274 __ aesenc(xmm_result, xmm_temp4);
2279 __ cmpl(keylen, 44);
2280 __ jccb(Assembler::equal, L_doLast);
2282 __ aesenc(xmm_result, xmm_temp1);
2283 __ aesenc(xmm_result, xmm_temp2);
2288 __ cmpl(keylen, 52);
2289 __ jccb(Assembler::equal, L_doLast);
2291 __ aesenc(xmm_result, xmm_temp1);
2292 __ aesenc(xmm_result, xmm_temp2);
2297 __ BIND(L_doLast);
2298 __ aesenc(xmm_result, xmm_temp1);
2299 __ aesenclast(xmm_result, xmm_temp2);
2300 __ movdqu(Address(to, 0), xmm_result); // store the result
2301 __ xorptr(rax, rax); // return 0
2302 __ leave(); // required for proper stackwalking of RuntimeStub frame
2303 __ ret(0);
2318 __ align(CodeEntryAlignment);
2321 address start = __ pc();
2338 __ enter(); // required for proper stackwalking of RuntimeStub frame
2339 __ movptr(from, from_param);
2340 __ movptr(key, key_param);
2343 __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2345 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2346 __ movdqu(xmm_result, Address(from, 0));
2347 __ movptr(to, to_param);
2357 __ pxor (xmm_result, xmm_temp1);
2358 __ aesdec(xmm_result, xmm_temp2);
2359 __ aesdec(xmm_result, xmm_temp3);
2360 __ aesdec(xmm_result, xmm_temp4);
2367 __ aesdec(xmm_result, xmm_temp1);
2368 __ aesdec(xmm_result, xmm_temp2);
2369 __ aesdec(xmm_result, xmm_temp3);
2370 __ aesdec(xmm_result, xmm_temp4);
2376 __ cmpl(keylen, 44);
2377 __ jccb(Assembler::equal, L_doLast);
2379 __ aesdec(xmm_result, xmm_temp1);
2380 __ aesdec(xmm_result, xmm_temp2);
2385 __ cmpl(keylen, 52);
2386 __ jccb(Assembler::equal, L_doLast);
2388 __ aesdec(xmm_result, xmm_temp1);
2389 __ aesdec(xmm_result, xmm_temp2);
2394 __ BIND(L_doLast);
2395 __ aesdec(xmm_result, xmm_temp1);
2396 __ aesdec(xmm_result, xmm_temp2);
2399 __ aesdeclast(xmm_result, xmm_temp3);
2400 __ movdqu(Address(to, 0), xmm_result); // store the result
2401 __ xorptr(rax, rax); // return 0
2402 __ leave(); // required for proper stackwalking of RuntimeStub frame
2403 __ ret(0);
2415 __ subptr(rsp, saveFrameSizeInBytes);
2416 __ movptr(saved_rsi, rsi);
2417 __ movptr(saved_rdi, rdi);
2418 __ movptr(saved_rbx, rbx);
2421 __ movptr(rsi, saved_rsi);
2422 __ movptr(rdi, saved_rdi);
2423 __ movptr(rbx, saved_rbx);
2438 __ align(CodeEntryAlignment);
2440 address start = __ pc();
2459 __ enter(); // required for proper stackwalking of RuntimeStub frame
2468 __ movptr(from , from_param);
2469 __ movptr(to , to_param);
2470 __ movptr(key , key_param);
2471 __ movptr(rvec , rvec_param);
2472 __ movptr(len_reg , len_param);
2475 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2482 __ movdqu(xmm_result, Address(rvec, 0x00)); // initialize xmm_result with r vec
2485 __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2486 __ cmpl(rax, 44);
2487 __ jcc(Assembler::notEqual, L_key_192_256);
2490 __ movl(pos, 0);
2491 __ align(OptoLoopAlignment);
2492 __ BIND(L_loopTop_128);
2493 __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
2494 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
2496 __ pxor (xmm_result, xmm_key0); // do the aes rounds
2498 __ aesenc(xmm_result, as_XMMRegister(rnum));
2504 __ aesenclast(xmm_result, xmm_temp);
2506 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
2508 __ addptr(pos, AESBlockSize);
2509 __ subptr(len_reg, AESBlockSize);
2510 __ jcc(Assembler::notEqual, L_loopTop_128);
2512 __ BIND(L_exit);
2513 __ movdqu(Address(rvec, 0), xmm_result); // final value of r stored in rvec of CipherBlockChaining object
2516 __ movl(rax, 0); // return 0 (why?)
2517 __ leave(); // required for proper stackwalking of RuntimeStub frame
2518 __ ret(0);
2520 __ BIND(L_key_192_256);
2522 __ cmpl(rax, 52);
2523 __ jcc(Assembler::notEqual, L_key_256);
2526 __ movl(pos, 0);
2527 __ align(OptoLoopAlignment);
2528 __ BIND(L_loopTop_192);
2529 __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
2530 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
2532 __ pxor (xmm_result, xmm_key0); // do the aes rounds
2534 __ aesenc(xmm_result, as_XMMRegister(rnum));
2540 __ aesenclast(xmm_result, xmm_temp);
2542 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
2544 __ addptr(pos, AESBlockSize);
2545 __ subptr(len_reg, AESBlockSize);
2546 __ jcc(Assembler::notEqual, L_loopTop_192);
2547 __ jmp(L_exit);
2549 __ BIND(L_key_256);
2551 __ movl(pos, 0);
2552 __ align(OptoLoopAlignment);
2553 __ BIND(L_loopTop_256);
2554 __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
2555 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
2557 __ pxor (xmm_result, xmm_key0); // do the aes rounds
2559 __ aesenc(xmm_result, as_XMMRegister(rnum));
2565 __ aesenclast(xmm_result, xmm_temp);
2567 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
2569 __ addptr(pos, AESBlockSize);
2570 __ subptr(len_reg, AESBlockSize);
2571 __ jcc(Assembler::notEqual, L_loopTop_256);
2572 __ jmp(L_exit);
2593 __ align(CodeEntryAlignment);
2595 address start = __ pc();
2617 __ enter(); // required for proper stackwalking of RuntimeStub frame
2626 __ movptr(from , from_param);
2627 __ movptr(to , to_param);
2628 __ movptr(key , key_param);
2629 __ movptr(rvec , rvec_param);
2630 __ movptr(len_reg , len_param);
2635 __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2647 __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2648 __ cmpl(rax, 44);
2649 __ jcc(Assembler::notEqual, L_key_192_256);
2653 __ movl(pos, 0);
2654 __ align(OptoLoopAlignment);
2655 __ BIND(L_singleBlock_loopTop_128);
2656 __ cmpptr(len_reg, 0); // any blocks left??
2657 __ jcc(Assembler::equal, L_exit);
2658 __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2659 __ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
2661 __ aesdec(xmm_result, as_XMMRegister(rnum));
2667 __ aesdeclast(xmm_result, xmm_temp);
2668 __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2669 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
2670 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
2672 __ lea(prev_block_cipher_ptr, Address(from, pos, Address::times_1, 0)); // set up new ptr
2673 __ addptr(pos, AESBlockSize);
2674 __ subptr(len_reg, AESBlockSize);
2675 __ jmp(L_singleBlock_loopTop_128);
2678 __ BIND(L_exit);
2679 __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2680 __ movptr(rvec , rvec_param); // restore this since used in loop
2681 __ movdqu(Address(rvec, 0), xmm_temp); // final value of r stored in rvec of CipherBlockChaining object
2683 __ movl(rax, 0); // return 0 (why?)
2684 __ leave(); // required for proper stackwalking of RuntimeStub frame
2685 __ ret(0);
2688 __ BIND(L_key_192_256);
2690 __ cmpl(rax, 52);
2691 __ jcc(Assembler::notEqual, L_key_256);
2694 __ movl(pos, 0);
2695 __ align(OptoLoopAlignment);
2696 __ BIND(L_singleBlock_loopTop_192);
2697 __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2698 __ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
2700 __ aesdec(xmm_result, as_XMMRegister(rnum));
2706 __ aesdeclast(xmm_result, xmm_temp);
2707 __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2708 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
2709 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
2711 __ lea(prev_block_cipher_ptr, Address(from, pos, Address::times_1, 0)); // set up new ptr
2712 __ addptr(pos, AESBlockSize);
2713 __ subptr(len_reg, AESBlockSize);
2714 __ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
2715 __ jmp(L_exit);
2717 __ BIND(L_key_256);
2719 __ movl(pos, 0);
2720 __ align(OptoLoopAlignment);
2721 __ BIND(L_singleBlock_loopTop_256);
2722 __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2723 __ pxor (xmm_result, xmm_key_first); // do the aes dec rounds
2725 __ aesdec(xmm_result, as_XMMRegister(rnum));
2731 __ aesdeclast(xmm_result, xmm_temp);
2732 __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2733 __ pxor (xmm_result, xmm_temp); // xor with the current r vector
2734 __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
2736 __ lea(prev_block_cipher_ptr, Address(from, pos, Address::times_1, 0)); // set up new ptr
2737 __ addptr(pos, AESBlockSize);
2738 __ subptr(len_reg, AESBlockSize);
2739 __ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
2740 __ jmp(L_exit);
2762 #undef __
2763 #define __ masm->
2798 address start = __ pc();
2805 __ get_thread(java_thread);
2807 __ enter(); // required for proper stackwalking of RuntimeStub frame
2810 __ subptr(rsp, (framesize-2) * wordSize); // prolog
2814 int frame_complete = __ pc() - start;
2817 __ movptr(Address(rsp, thread_off * wordSize), java_thread);
2819 __ movptr(Address(rsp, arg1_off * wordSize), arg1);
2823 __ movptr(Address(rsp, arg2_off * wordSize), arg2);
2827 __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
2831 __ call(RuntimeAddress(runtime_entry));
2834 oop_maps->add_gc_map(__ pc() - start, map);
2839 __ get_thread(java_thread);
2841 __ reset_last_Java_frame(java_thread, true, false);
2843 __ leave(); // required for proper stackwalking of RuntimeStub frame
2848 __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
2849 __ jcc(Assembler::notEqual, L);
2850 __ should_not_reach_here();
2851 __ bind(L);
2853 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));