Text file
src/runtime/asm_arm64.s
Documentation: runtime
1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "tls_arm64.h"
8 #include "funcdata.h"
9 #include "textflag.h"
10
11 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
12 // SP = stack; R0 = argc; R1 = argv
13
14 SUB $32, RSP
15 MOVW R0, 8(RSP) // argc
16 MOVD R1, 16(RSP) // argv
17
18 #ifdef TLS_darwin
19 // Initialize TLS.
20 MOVD ZR, g // clear g, make sure it's not junk.
21 SUB $32, RSP
22 MRS_TPIDR_R0
23 AND $~7, R0
24 MOVD R0, 16(RSP) // arg2: TLS base
25 MOVD $runtime·tls_g(SB), R2
26 MOVD R2, 8(RSP) // arg1: &tlsg
27 BL ·tlsinit(SB)
28 ADD $32, RSP
29 #endif
30
31 // create istack out of the given (operating system) stack.
32 // _cgo_init may update stackguard.
33 MOVD $runtime·g0(SB), g
34 MOVD RSP, R7
35 MOVD $(-64*1024)(R7), R0
36 MOVD R0, g_stackguard0(g)
37 MOVD R0, g_stackguard1(g)
38 MOVD R0, (g_stack+stack_lo)(g)
39 MOVD R7, (g_stack+stack_hi)(g)
40
41 // if there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CBZ R12, nocgo
44
45 #ifdef GOOS_android
46 MRS_TPIDR_R0 // load TLS base pointer
47 MOVD R0, R3 // arg 3: TLS base pointer
48 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g
49 #else
50 MOVD $0, R2 // arg 2: not used when using platform's TLS
51 #endif
52 MOVD $setg_gcc<>(SB), R1 // arg 1: setg
53 MOVD g, R0 // arg 0: G
54 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved.
55 BL (R12)
56 ADD $16, RSP
57
58 nocgo:
59 BL runtime·save_g(SB)
60 // update stackguard after _cgo_init
61 MOVD (g_stack+stack_lo)(g), R0
62 ADD $const__StackGuard, R0
63 MOVD R0, g_stackguard0(g)
64 MOVD R0, g_stackguard1(g)
65
66 // set the per-goroutine and per-mach "registers"
67 MOVD $runtime·m0(SB), R0
68
69 // save m->g0 = g0
70 MOVD g, m_g0(R0)
71 // save m0 to g0->m
72 MOVD R0, g_m(g)
73
74 BL runtime·check(SB)
75
76 #ifdef GOOS_windows
77 BL runtime·wintls(SB)
78 #endif
79
80 MOVW 8(RSP), R0 // copy argc
81 MOVW R0, -8(RSP)
82 MOVD 16(RSP), R0 // copy argv
83 MOVD R0, 0(RSP)
84 BL runtime·args(SB)
85 BL runtime·osinit(SB)
86 BL runtime·schedinit(SB)
87
88 // create a new goroutine to start program
89 MOVD $runtime·mainPC(SB), R0 // entry
90 MOVD RSP, R7
91 MOVD.W $0, -8(R7)
92 MOVD.W R0, -8(R7)
93 MOVD.W $0, -8(R7)
94 MOVD.W $0, -8(R7)
95 MOVD R7, RSP
96 BL runtime·newproc(SB)
97 ADD $32, RSP
98
99 // start this M
100 BL runtime·mstart(SB)
101
102 MOVD $0, R0
103 MOVD R0, (R0) // boom
104 UNDEF
105
106 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
107 GLOBL runtime·mainPC(SB),RODATA,$8
108
109 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
110 BRK
111 RET
112
113 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
114 RET
115
116 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
117 BL runtime·mstart0(SB)
118 RET // not reached
119
120 /*
121 * go-routine
122 */
123
124 // void gogo(Gobuf*)
125 // restore state from Gobuf; longjmp
126 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
127 MOVD buf+0(FP), R5
128 MOVD gobuf_g(R5), R6
129 MOVD 0(R6), R4 // make sure g != nil
130 B gogo<>(SB)
131
132 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
133 MOVD R6, g
134 BL runtime·save_g(SB)
135
136 MOVD gobuf_sp(R5), R0
137 MOVD R0, RSP
138 MOVD gobuf_bp(R5), R29
139 MOVD gobuf_lr(R5), LR
140 MOVD gobuf_ret(R5), R0
141 MOVD gobuf_ctxt(R5), R26
142 MOVD $0, gobuf_sp(R5)
143 MOVD $0, gobuf_bp(R5)
144 MOVD $0, gobuf_ret(R5)
145 MOVD $0, gobuf_lr(R5)
146 MOVD $0, gobuf_ctxt(R5)
147 CMP ZR, ZR // set condition codes for == test, needed by stack split
148 MOVD gobuf_pc(R5), R6
149 B (R6)
150
151 // void mcall(fn func(*g))
152 // Switch to m->g0's stack, call fn(g).
153 // Fn must never return. It should gogo(&g->sched)
154 // to keep running g.
155 TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
156 // Save caller state in g->sched
157 MOVD RSP, R0
158 MOVD R0, (g_sched+gobuf_sp)(g)
159 MOVD R29, (g_sched+gobuf_bp)(g)
160 MOVD LR, (g_sched+gobuf_pc)(g)
161 MOVD $0, (g_sched+gobuf_lr)(g)
162
163 // Switch to m->g0 & its stack, call fn.
164 MOVD g, R3
165 MOVD g_m(g), R8
166 MOVD m_g0(R8), g
167 BL runtime·save_g(SB)
168 CMP g, R3
169 BNE 2(PC)
170 B runtime·badmcall(SB)
171 MOVD fn+0(FP), R26 // context
172 MOVD 0(R26), R4 // code pointer
173 MOVD (g_sched+gobuf_sp)(g), R0
174 MOVD R0, RSP // sp = m->g0->sched.sp
175 MOVD (g_sched+gobuf_bp)(g), R29
176 MOVD R3, -8(RSP)
177 MOVD $0, -16(RSP)
178 SUB $16, RSP
179 BL (R4)
180 B runtime·badmcall2(SB)
181
182 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
183 // of the G stack. We need to distinguish the routine that
184 // lives at the bottom of the G stack from the one that lives
185 // at the top of the system stack because the one at the top of
186 // the system stack terminates the stack walk (see topofstack()).
187 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
188 UNDEF
189 BL (LR) // make sure this function is not leaf
190 RET
191
192 // func systemstack(fn func())
193 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
194 MOVD fn+0(FP), R3 // R3 = fn
195 MOVD R3, R26 // context
196 MOVD g_m(g), R4 // R4 = m
197
198 MOVD m_gsignal(R4), R5 // R5 = gsignal
199 CMP g, R5
200 BEQ noswitch
201
202 MOVD m_g0(R4), R5 // R5 = g0
203 CMP g, R5
204 BEQ noswitch
205
206 MOVD m_curg(R4), R6
207 CMP g, R6
208 BEQ switch
209
210 // Bad: g is not gsignal, not g0, not curg. What is it?
211 // Hide call from linker nosplit analysis.
212 MOVD $runtime·badsystemstack(SB), R3
213 BL (R3)
214 B runtime·abort(SB)
215
216 switch:
217 // save our state in g->sched. Pretend to
218 // be systemstack_switch if the G stack is scanned.
219 BL gosave_systemstack_switch<>(SB)
220
221 // switch to g0
222 MOVD R5, g
223 BL runtime·save_g(SB)
224 MOVD (g_sched+gobuf_sp)(g), R3
225 MOVD R3, RSP
226 MOVD (g_sched+gobuf_bp)(g), R29
227
228 // call target function
229 MOVD 0(R26), R3 // code pointer
230 BL (R3)
231
232 // switch back to g
233 MOVD g_m(g), R3
234 MOVD m_curg(R3), g
235 BL runtime·save_g(SB)
236 MOVD (g_sched+gobuf_sp)(g), R0
237 MOVD R0, RSP
238 MOVD (g_sched+gobuf_bp)(g), R29
239 MOVD $0, (g_sched+gobuf_sp)(g)
240 MOVD $0, (g_sched+gobuf_bp)(g)
241 RET
242
243 noswitch:
244 // already on m stack, just call directly
245 // Using a tail call here cleans up tracebacks since we won't stop
246 // at an intermediate systemstack.
247 MOVD 0(R26), R3 // code pointer
248 MOVD.P 16(RSP), R30 // restore LR
249 SUB $8, RSP, R29 // restore FP
250 B (R3)
251
252 /*
253 * support for morestack
254 */
255
256 // Called during function prolog when more stack is needed.
257 // Caller has already loaded:
258 // R3 prolog's LR (R30)
259 //
260 // The traceback routines see morestack on a g0 as being
261 // the top of a stack (for example, morestack calling newstack
262 // calling the scheduler calling newm calling gc), so we must
263 // record an argument size. For that purpose, it has no arguments.
264 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
265 // Cannot grow scheduler stack (m->g0).
266 MOVD g_m(g), R8
267 MOVD m_g0(R8), R4
268 CMP g, R4
269 BNE 3(PC)
270 BL runtime·badmorestackg0(SB)
271 B runtime·abort(SB)
272
273 // Cannot grow signal stack (m->gsignal).
274 MOVD m_gsignal(R8), R4
275 CMP g, R4
276 BNE 3(PC)
277 BL runtime·badmorestackgsignal(SB)
278 B runtime·abort(SB)
279
280 // Called from f.
281 // Set g->sched to context in f
282 MOVD RSP, R0
283 MOVD R0, (g_sched+gobuf_sp)(g)
284 MOVD R29, (g_sched+gobuf_bp)(g)
285 MOVD LR, (g_sched+gobuf_pc)(g)
286 MOVD R3, (g_sched+gobuf_lr)(g)
287 MOVD R26, (g_sched+gobuf_ctxt)(g)
288
289 // Called from f.
290 // Set m->morebuf to f's callers.
291 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
292 MOVD RSP, R0
293 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP
294 MOVD g, (m_morebuf+gobuf_g)(R8)
295
296 // Call newstack on m->g0's stack.
297 MOVD m_g0(R8), g
298 BL runtime·save_g(SB)
299 MOVD (g_sched+gobuf_sp)(g), R0
300 MOVD R0, RSP
301 MOVD (g_sched+gobuf_bp)(g), R29
302 MOVD.W $0, -16(RSP) // create a call frame on g0 (saved LR; keep 16-aligned)
303 BL runtime·newstack(SB)
304
305 // Not reached, but make sure the return PC from the call to newstack
306 // is still in this function, and not the beginning of the next.
307 UNDEF
308
309 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
310 MOVW $0, R26
311 B runtime·morestack(SB)
312
313 // reflectcall: call a function with the given argument list
314 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
315 // we don't have variable-sized frames, so we use a small number
316 // of constant-sized-frame functions to encode a few bits of size in the pc.
317 // Caution: ugly multiline assembly macros in your future!
318
319 #define DISPATCH(NAME,MAXSIZE) \
320 MOVD $MAXSIZE, R27; \
321 CMP R27, R16; \
322 BGT 3(PC); \
323 MOVD $NAME(SB), R27; \
324 B (R27)
325 // Note: can't just "B NAME(SB)" - bad inlining results.
326
327 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
328 MOVWU frameSize+32(FP), R16
329 DISPATCH(runtime·call16, 16)
330 DISPATCH(runtime·call32, 32)
331 DISPATCH(runtime·call64, 64)
332 DISPATCH(runtime·call128, 128)
333 DISPATCH(runtime·call256, 256)
334 DISPATCH(runtime·call512, 512)
335 DISPATCH(runtime·call1024, 1024)
336 DISPATCH(runtime·call2048, 2048)
337 DISPATCH(runtime·call4096, 4096)
338 DISPATCH(runtime·call8192, 8192)
339 DISPATCH(runtime·call16384, 16384)
340 DISPATCH(runtime·call32768, 32768)
341 DISPATCH(runtime·call65536, 65536)
342 DISPATCH(runtime·call131072, 131072)
343 DISPATCH(runtime·call262144, 262144)
344 DISPATCH(runtime·call524288, 524288)
345 DISPATCH(runtime·call1048576, 1048576)
346 DISPATCH(runtime·call2097152, 2097152)
347 DISPATCH(runtime·call4194304, 4194304)
348 DISPATCH(runtime·call8388608, 8388608)
349 DISPATCH(runtime·call16777216, 16777216)
350 DISPATCH(runtime·call33554432, 33554432)
351 DISPATCH(runtime·call67108864, 67108864)
352 DISPATCH(runtime·call134217728, 134217728)
353 DISPATCH(runtime·call268435456, 268435456)
354 DISPATCH(runtime·call536870912, 536870912)
355 DISPATCH(runtime·call1073741824, 1073741824)
356 MOVD $runtime·badreflectcall(SB), R0
357 B (R0)
358
359 #define CALLFN(NAME,MAXSIZE) \
360 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
361 NO_LOCAL_POINTERS; \
362 /* copy arguments to stack */ \
363 MOVD stackArgs+16(FP), R3; \
364 MOVWU stackArgsSize+24(FP), R4; \
365 ADD $8, RSP, R5; \
366 BIC $0xf, R4, R6; \
367 CBZ R6, 6(PC); \
368 /* if R6=(argsize&~15) != 0 */ \
369 ADD R6, R5, R6; \
370 /* copy 16 bytes a time */ \
371 LDP.P 16(R3), (R7, R8); \
372 STP.P (R7, R8), 16(R5); \
373 CMP R5, R6; \
374 BNE -3(PC); \
375 AND $0xf, R4, R6; \
376 CBZ R6, 6(PC); \
377 /* if R6=(argsize&15) != 0 */ \
378 ADD R6, R5, R6; \
379 /* copy 1 byte a time for the rest */ \
380 MOVBU.P 1(R3), R7; \
381 MOVBU.P R7, 1(R5); \
382 CMP R5, R6; \
383 BNE -3(PC); \
384 /* call function */ \
385 MOVD f+8(FP), R26; \
386 MOVD (R26), R0; \
387 PCDATA $PCDATA_StackMapIndex, $0; \
388 BL (R0); \
389 /* copy return values back */ \
390 MOVD stackArgsType+0(FP), R7; \
391 MOVD stackArgs+16(FP), R3; \
392 MOVWU stackArgsSize+24(FP), R4; \
393 MOVWU stackRetOffset+28(FP), R6; \
394 ADD $8, RSP, R5; \
395 ADD R6, R5; \
396 ADD R6, R3; \
397 SUB R6, R4; \
398 BL callRet<>(SB); \
399 RET
400
401 // callRet copies return values back at the end of call*. This is a
402 // separate function so it can allocate stack space for the arguments
403 // to reflectcallmove. It does not follow the Go ABI; it expects its
404 // arguments in registers.
405 TEXT callRet<>(SB), NOSPLIT, $48-0
406 MOVD R7, 8(RSP)
407 MOVD R3, 16(RSP)
408 MOVD R5, 24(RSP)
409 MOVD R4, 32(RSP)
410 MOVD $0, 40(RSP)
411 BL runtime·reflectcallmove(SB)
412 RET
413
414 CALLFN(·call16, 16)
415 CALLFN(·call32, 32)
416 CALLFN(·call64, 64)
417 CALLFN(·call128, 128)
418 CALLFN(·call256, 256)
419 CALLFN(·call512, 512)
420 CALLFN(·call1024, 1024)
421 CALLFN(·call2048, 2048)
422 CALLFN(·call4096, 4096)
423 CALLFN(·call8192, 8192)
424 CALLFN(·call16384, 16384)
425 CALLFN(·call32768, 32768)
426 CALLFN(·call65536, 65536)
427 CALLFN(·call131072, 131072)
428 CALLFN(·call262144, 262144)
429 CALLFN(·call524288, 524288)
430 CALLFN(·call1048576, 1048576)
431 CALLFN(·call2097152, 2097152)
432 CALLFN(·call4194304, 4194304)
433 CALLFN(·call8388608, 8388608)
434 CALLFN(·call16777216, 16777216)
435 CALLFN(·call33554432, 33554432)
436 CALLFN(·call67108864, 67108864)
437 CALLFN(·call134217728, 134217728)
438 CALLFN(·call268435456, 268435456)
439 CALLFN(·call536870912, 536870912)
440 CALLFN(·call1073741824, 1073741824)
441
442 // func memhash32(p unsafe.Pointer, h uintptr) uintptr
443 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
444 MOVB runtime·useAeshash(SB), R0
445 CBZ R0, noaes
446 MOVD p+0(FP), R0
447 MOVD h+8(FP), R1
448 MOVD $ret+16(FP), R2
449 MOVD $runtime·aeskeysched+0(SB), R3
450
451 VEOR V0.B16, V0.B16, V0.B16
452 VLD1 (R3), [V2.B16]
453 VLD1 (R0), V0.S[1]
454 VMOV R1, V0.S[0]
455
456 AESE V2.B16, V0.B16
457 AESMC V0.B16, V0.B16
458 AESE V2.B16, V0.B16
459 AESMC V0.B16, V0.B16
460 AESE V2.B16, V0.B16
461
462 VST1 [V0.D1], (R2)
463 RET
464 noaes:
465 B runtime·memhash32Fallback(SB)
466
467 // func memhash64(p unsafe.Pointer, h uintptr) uintptr
468 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
469 MOVB runtime·useAeshash(SB), R0
470 CBZ R0, noaes
471 MOVD p+0(FP), R0
472 MOVD h+8(FP), R1
473 MOVD $ret+16(FP), R2
474 MOVD $runtime·aeskeysched+0(SB), R3
475
476 VEOR V0.B16, V0.B16, V0.B16
477 VLD1 (R3), [V2.B16]
478 VLD1 (R0), V0.D[1]
479 VMOV R1, V0.D[0]
480
481 AESE V2.B16, V0.B16
482 AESMC V0.B16, V0.B16
483 AESE V2.B16, V0.B16
484 AESMC V0.B16, V0.B16
485 AESE V2.B16, V0.B16
486
487 VST1 [V0.D1], (R2)
488 RET
489 noaes:
490 B runtime·memhash64Fallback(SB)
491
492 // func memhash(p unsafe.Pointer, h, size uintptr) uintptr
493 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
494 MOVB runtime·useAeshash(SB), R0
495 CBZ R0, noaes
496 MOVD p+0(FP), R0
497 MOVD s+16(FP), R1
498 MOVD h+8(FP), R3
499 MOVD $ret+24(FP), R2
500 B aeshashbody<>(SB)
501 noaes:
502 B runtime·memhashFallback(SB)
503
504 // func strhash(p unsafe.Pointer, h uintptr) uintptr
505 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
506 MOVB runtime·useAeshash(SB), R0
507 CBZ R0, noaes
508 MOVD p+0(FP), R10 // string pointer
509 LDP (R10), (R0, R1) //string data/ length
510 MOVD h+8(FP), R3
511 MOVD $ret+16(FP), R2 // return adddress
512 B aeshashbody<>(SB)
513 noaes:
514 B runtime·strhashFallback(SB)
515
516 // R0: data
517 // R1: length
518 // R2: address to put return value
519 // R3: seed data
520 TEXT aeshashbody<>(SB),NOSPLIT|NOFRAME,$0
521 VEOR V30.B16, V30.B16, V30.B16
522 VMOV R3, V30.D[0]
523 VMOV R1, V30.D[1] // load length into seed
524
525 MOVD $runtime·aeskeysched+0(SB), R4
526 VLD1.P 16(R4), [V0.B16]
527 AESE V30.B16, V0.B16
528 AESMC V0.B16, V0.B16
529 CMP $16, R1
530 BLO aes0to15
531 BEQ aes16
532 CMP $32, R1
533 BLS aes17to32
534 CMP $64, R1
535 BLS aes33to64
536 CMP $128, R1
537 BLS aes65to128
538 B aes129plus
539
540 aes0to15:
541 CBZ R1, aes0
542 VEOR V2.B16, V2.B16, V2.B16
543 TBZ $3, R1, less_than_8
544 VLD1.P 8(R0), V2.D[0]
545
546 less_than_8:
547 TBZ $2, R1, less_than_4
548 VLD1.P 4(R0), V2.S[2]
549
550 less_than_4:
551 TBZ $1, R1, less_than_2
552 VLD1.P 2(R0), V2.H[6]
553
554 less_than_2:
555 TBZ $0, R1, done
556 VLD1 (R0), V2.B[14]
557 done:
558 AESE V0.B16, V2.B16
559 AESMC V2.B16, V2.B16
560 AESE V0.B16, V2.B16
561 AESMC V2.B16, V2.B16
562 AESE V0.B16, V2.B16
563
564 VST1 [V2.D1], (R2)
565 RET
566 aes0:
567 VST1 [V0.D1], (R2)
568 RET
569 aes16:
570 VLD1 (R0), [V2.B16]
571 B done
572
573 aes17to32:
574 // make second seed
575 VLD1 (R4), [V1.B16]
576 AESE V30.B16, V1.B16
577 AESMC V1.B16, V1.B16
578 SUB $16, R1, R10
579 VLD1.P (R0)(R10), [V2.B16]
580 VLD1 (R0), [V3.B16]
581
582 AESE V0.B16, V2.B16
583 AESMC V2.B16, V2.B16
584 AESE V1.B16, V3.B16
585 AESMC V3.B16, V3.B16
586
587 AESE V0.B16, V2.B16
588 AESMC V2.B16, V2.B16
589 AESE V1.B16, V3.B16
590 AESMC V3.B16, V3.B16
591
592 AESE V0.B16, V2.B16
593 AESE V1.B16, V3.B16
594
595 VEOR V3.B16, V2.B16, V2.B16
596 VST1 [V2.D1], (R2)
597 RET
598
599 aes33to64:
600 VLD1 (R4), [V1.B16, V2.B16, V3.B16]
601 AESE V30.B16, V1.B16
602 AESMC V1.B16, V1.B16
603 AESE V30.B16, V2.B16
604 AESMC V2.B16, V2.B16
605 AESE V30.B16, V3.B16
606 AESMC V3.B16, V3.B16
607 SUB $32, R1, R10
608
609 VLD1.P (R0)(R10), [V4.B16, V5.B16]
610 VLD1 (R0), [V6.B16, V7.B16]
611
612 AESE V0.B16, V4.B16
613 AESMC V4.B16, V4.B16
614 AESE V1.B16, V5.B16
615 AESMC V5.B16, V5.B16
616 AESE V2.B16, V6.B16
617 AESMC V6.B16, V6.B16
618 AESE V3.B16, V7.B16
619 AESMC V7.B16, V7.B16
620
621 AESE V0.B16, V4.B16
622 AESMC V4.B16, V4.B16
623 AESE V1.B16, V5.B16
624 AESMC V5.B16, V5.B16
625 AESE V2.B16, V6.B16
626 AESMC V6.B16, V6.B16
627 AESE V3.B16, V7.B16
628 AESMC V7.B16, V7.B16
629
630 AESE V0.B16, V4.B16
631 AESE V1.B16, V5.B16
632 AESE V2.B16, V6.B16
633 AESE V3.B16, V7.B16
634
635 VEOR V6.B16, V4.B16, V4.B16
636 VEOR V7.B16, V5.B16, V5.B16
637 VEOR V5.B16, V4.B16, V4.B16
638
639 VST1 [V4.D1], (R2)
640 RET
641
642 aes65to128:
643 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
644 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
645 AESE V30.B16, V1.B16
646 AESMC V1.B16, V1.B16
647 AESE V30.B16, V2.B16
648 AESMC V2.B16, V2.B16
649 AESE V30.B16, V3.B16
650 AESMC V3.B16, V3.B16
651 AESE V30.B16, V4.B16
652 AESMC V4.B16, V4.B16
653 AESE V30.B16, V5.B16
654 AESMC V5.B16, V5.B16
655 AESE V30.B16, V6.B16
656 AESMC V6.B16, V6.B16
657 AESE V30.B16, V7.B16
658 AESMC V7.B16, V7.B16
659
660 SUB $64, R1, R10
661 VLD1.P (R0)(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
662 VLD1 (R0), [V12.B16, V13.B16, V14.B16, V15.B16]
663 AESE V0.B16, V8.B16
664 AESMC V8.B16, V8.B16
665 AESE V1.B16, V9.B16
666 AESMC V9.B16, V9.B16
667 AESE V2.B16, V10.B16
668 AESMC V10.B16, V10.B16
669 AESE V3.B16, V11.B16
670 AESMC V11.B16, V11.B16
671 AESE V4.B16, V12.B16
672 AESMC V12.B16, V12.B16
673 AESE V5.B16, V13.B16
674 AESMC V13.B16, V13.B16
675 AESE V6.B16, V14.B16
676 AESMC V14.B16, V14.B16
677 AESE V7.B16, V15.B16
678 AESMC V15.B16, V15.B16
679
680 AESE V0.B16, V8.B16
681 AESMC V8.B16, V8.B16
682 AESE V1.B16, V9.B16
683 AESMC V9.B16, V9.B16
684 AESE V2.B16, V10.B16
685 AESMC V10.B16, V10.B16
686 AESE V3.B16, V11.B16
687 AESMC V11.B16, V11.B16
688 AESE V4.B16, V12.B16
689 AESMC V12.B16, V12.B16
690 AESE V5.B16, V13.B16
691 AESMC V13.B16, V13.B16
692 AESE V6.B16, V14.B16
693 AESMC V14.B16, V14.B16
694 AESE V7.B16, V15.B16
695 AESMC V15.B16, V15.B16
696
697 AESE V0.B16, V8.B16
698 AESE V1.B16, V9.B16
699 AESE V2.B16, V10.B16
700 AESE V3.B16, V11.B16
701 AESE V4.B16, V12.B16
702 AESE V5.B16, V13.B16
703 AESE V6.B16, V14.B16
704 AESE V7.B16, V15.B16
705
706 VEOR V12.B16, V8.B16, V8.B16
707 VEOR V13.B16, V9.B16, V9.B16
708 VEOR V14.B16, V10.B16, V10.B16
709 VEOR V15.B16, V11.B16, V11.B16
710 VEOR V10.B16, V8.B16, V8.B16
711 VEOR V11.B16, V9.B16, V9.B16
712 VEOR V9.B16, V8.B16, V8.B16
713
714 VST1 [V8.D1], (R2)
715 RET
716
717 aes129plus:
718 PRFM (R0), PLDL1KEEP
719 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
720 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
721 AESE V30.B16, V1.B16
722 AESMC V1.B16, V1.B16
723 AESE V30.B16, V2.B16
724 AESMC V2.B16, V2.B16
725 AESE V30.B16, V3.B16
726 AESMC V3.B16, V3.B16
727 AESE V30.B16, V4.B16
728 AESMC V4.B16, V4.B16
729 AESE V30.B16, V5.B16
730 AESMC V5.B16, V5.B16
731 AESE V30.B16, V6.B16
732 AESMC V6.B16, V6.B16
733 AESE V30.B16, V7.B16
734 AESMC V7.B16, V7.B16
735 ADD R0, R1, R10
736 SUB $128, R10, R10
737 VLD1.P 64(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
738 VLD1 (R10), [V12.B16, V13.B16, V14.B16, V15.B16]
739 SUB $1, R1, R1
740 LSR $7, R1, R1
741
742 aesloop:
743 AESE V8.B16, V0.B16
744 AESMC V0.B16, V0.B16
745 AESE V9.B16, V1.B16
746 AESMC V1.B16, V1.B16
747 AESE V10.B16, V2.B16
748 AESMC V2.B16, V2.B16
749 AESE V11.B16, V3.B16
750 AESMC V3.B16, V3.B16
751 AESE V12.B16, V4.B16
752 AESMC V4.B16, V4.B16
753 AESE V13.B16, V5.B16
754 AESMC V5.B16, V5.B16
755 AESE V14.B16, V6.B16
756 AESMC V6.B16, V6.B16
757 AESE V15.B16, V7.B16
758 AESMC V7.B16, V7.B16
759
760 VLD1.P 64(R0), [V8.B16, V9.B16, V10.B16, V11.B16]
761 AESE V8.B16, V0.B16
762 AESMC V0.B16, V0.B16
763 AESE V9.B16, V1.B16
764 AESMC V1.B16, V1.B16
765 AESE V10.B16, V2.B16
766 AESMC V2.B16, V2.B16
767 AESE V11.B16, V3.B16
768 AESMC V3.B16, V3.B16
769
770 VLD1.P 64(R0), [V12.B16, V13.B16, V14.B16, V15.B16]
771 AESE V12.B16, V4.B16
772 AESMC V4.B16, V4.B16
773 AESE V13.B16, V5.B16
774 AESMC V5.B16, V5.B16
775 AESE V14.B16, V6.B16
776 AESMC V6.B16, V6.B16
777 AESE V15.B16, V7.B16
778 AESMC V7.B16, V7.B16
779 SUB $1, R1, R1
780 CBNZ R1, aesloop
781
782 AESE V8.B16, V0.B16
783 AESMC V0.B16, V0.B16
784 AESE V9.B16, V1.B16
785 AESMC V1.B16, V1.B16
786 AESE V10.B16, V2.B16
787 AESMC V2.B16, V2.B16
788 AESE V11.B16, V3.B16
789 AESMC V3.B16, V3.B16
790 AESE V12.B16, V4.B16
791 AESMC V4.B16, V4.B16
792 AESE V13.B16, V5.B16
793 AESMC V5.B16, V5.B16
794 AESE V14.B16, V6.B16
795 AESMC V6.B16, V6.B16
796 AESE V15.B16, V7.B16
797 AESMC V7.B16, V7.B16
798
799 AESE V8.B16, V0.B16
800 AESMC V0.B16, V0.B16
801 AESE V9.B16, V1.B16
802 AESMC V1.B16, V1.B16
803 AESE V10.B16, V2.B16
804 AESMC V2.B16, V2.B16
805 AESE V11.B16, V3.B16
806 AESMC V3.B16, V3.B16
807 AESE V12.B16, V4.B16
808 AESMC V4.B16, V4.B16
809 AESE V13.B16, V5.B16
810 AESMC V5.B16, V5.B16
811 AESE V14.B16, V6.B16
812 AESMC V6.B16, V6.B16
813 AESE V15.B16, V7.B16
814 AESMC V7.B16, V7.B16
815
816 AESE V8.B16, V0.B16
817 AESE V9.B16, V1.B16
818 AESE V10.B16, V2.B16
819 AESE V11.B16, V3.B16
820 AESE V12.B16, V4.B16
821 AESE V13.B16, V5.B16
822 AESE V14.B16, V6.B16
823 AESE V15.B16, V7.B16
824
825 VEOR V0.B16, V1.B16, V0.B16
826 VEOR V2.B16, V3.B16, V2.B16
827 VEOR V4.B16, V5.B16, V4.B16
828 VEOR V6.B16, V7.B16, V6.B16
829 VEOR V0.B16, V2.B16, V0.B16
830 VEOR V4.B16, V6.B16, V4.B16
831 VEOR V4.B16, V0.B16, V0.B16
832
833 VST1 [V0.D1], (R2)
834 RET
835
836 TEXT runtime·procyield(SB),NOSPLIT,$0-0
837 MOVWU cycles+0(FP), R0
838 again:
839 YIELD
840 SUBW $1, R0
841 CBNZ R0, again
842 RET
843
844 // void jmpdefer(fv, sp);
845 // called from deferreturn.
846 // 1. grab stored LR for caller
847 // 2. sub 4 bytes to get back to BL deferreturn
848 // 3. BR to fn
849 TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
850 MOVD 0(RSP), R0
851 SUB $4, R0
852 MOVD R0, LR
853
854 MOVD fv+0(FP), R26
855 MOVD argp+8(FP), R0
856 MOVD R0, RSP
857 SUB $8, RSP
858 MOVD 0(R26), R3
859 B (R3)
860
861 // Save state of caller into g->sched,
862 // but using fake PC from systemstack_switch.
863 // Must only be called from functions with no locals ($0)
864 // or else unwinding from systemstack_switch is incorrect.
865 // Smashes R0.
866 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
867 MOVD $runtime·systemstack_switch(SB), R0
868 ADD $8, R0 // get past prologue
869 MOVD R0, (g_sched+gobuf_pc)(g)
870 MOVD RSP, R0
871 MOVD R0, (g_sched+gobuf_sp)(g)
872 MOVD R29, (g_sched+gobuf_bp)(g)
873 MOVD $0, (g_sched+gobuf_lr)(g)
874 MOVD $0, (g_sched+gobuf_ret)(g)
875 // Assert ctxt is zero. See func save.
876 MOVD (g_sched+gobuf_ctxt)(g), R0
877 CBZ R0, 2(PC)
878 CALL runtime·abort(SB)
879 RET
880
881 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
882 // Call fn(arg) aligned appropriately for the gcc ABI.
883 // Called on a system stack, and there may be no g yet (during needm).
884 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
885 MOVD fn+0(FP), R1
886 MOVD arg+8(FP), R0
887 SUB $16, RSP // skip over saved frame pointer below RSP
888 BL (R1)
889 ADD $16, RSP // skip over saved frame pointer below RSP
890 RET
891
892 // func asmcgocall(fn, arg unsafe.Pointer) int32
893 // Call fn(arg) on the scheduler stack,
894 // aligned appropriately for the gcc ABI.
895 // See cgocall.go for more details.
896 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
897 MOVD fn+0(FP), R1
898 MOVD arg+8(FP), R0
899
900 MOVD RSP, R2 // save original stack pointer
901 CBZ g, nosave
902 MOVD g, R4
903
904 // Figure out if we need to switch to m->g0 stack.
905 // We get called to create new OS threads too, and those
906 // come in on the m->g0 stack already.
907 MOVD g_m(g), R8
908 MOVD m_gsignal(R8), R3
909 CMP R3, g
910 BEQ nosave
911 MOVD m_g0(R8), R3
912 CMP R3, g
913 BEQ nosave
914
915 // Switch to system stack.
916 MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0
917 BL gosave_systemstack_switch<>(SB)
918 MOVD R3, g
919 BL runtime·save_g(SB)
920 MOVD (g_sched+gobuf_sp)(g), R0
921 MOVD R0, RSP
922 MOVD (g_sched+gobuf_bp)(g), R29
923 MOVD R9, R0
924
925 // Now on a scheduling stack (a pthread-created stack).
926 // Save room for two of our pointers /*, plus 32 bytes of callee
927 // save area that lives on the caller stack. */
928 MOVD RSP, R13
929 SUB $16, R13
930 MOVD R13, RSP
931 MOVD R4, 0(RSP) // save old g on stack
932 MOVD (g_stack+stack_hi)(R4), R4
933 SUB R2, R4
934 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
935 BL (R1)
936 MOVD R0, R9
937
938 // Restore g, stack pointer. R0 is errno, so don't touch it
939 MOVD 0(RSP), g
940 BL runtime·save_g(SB)
941 MOVD (g_stack+stack_hi)(g), R5
942 MOVD 8(RSP), R6
943 SUB R6, R5
944 MOVD R9, R0
945 MOVD R5, RSP
946
947 MOVW R0, ret+16(FP)
948 RET
949
950 nosave:
951 // Running on a system stack, perhaps even without a g.
952 // Having no g can happen during thread creation or thread teardown
953 // (see needm/dropm on Solaris, for example).
954 // This code is like the above sequence but without saving/restoring g
955 // and without worrying about the stack moving out from under us
956 // (because we're on a system stack, not a goroutine stack).
957 // The above code could be used directly if already on a system stack,
958 // but then the only path through this code would be a rare case on Solaris.
959 // Using this code for all "already on system stack" calls exercises it more,
960 // which should help keep it correct.
961 MOVD RSP, R13
962 SUB $16, R13
963 MOVD R13, RSP
964 MOVD $0, R4
965 MOVD R4, 0(RSP) // Where above code stores g, in case someone looks during debugging.
966 MOVD R2, 8(RSP) // Save original stack pointer.
967 BL (R1)
968 // Restore stack pointer.
969 MOVD 8(RSP), R2
970 MOVD R2, RSP
971 MOVD R0, ret+16(FP)
972 RET
973
974 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
975 // See cgocall.go for more details.
976 TEXT ·cgocallback(SB),NOSPLIT,$24-24
977 NO_LOCAL_POINTERS
978
979 // Load g from thread-local storage.
980 BL runtime·load_g(SB)
981
982 // If g is nil, Go did not create the current thread.
983 // Call needm to obtain one for temporary use.
984 // In this case, we're running on the thread stack, so there's
985 // lots of space, but the linker doesn't know. Hide the call from
986 // the linker analysis by using an indirect call.
987 CBZ g, needm
988
989 MOVD g_m(g), R8
990 MOVD R8, savedm-8(SP)
991 B havem
992
993 needm:
994 MOVD g, savedm-8(SP) // g is zero, so is m.
995 MOVD $runtime·needm(SB), R0
996 BL (R0)
997
998 // Set m->g0->sched.sp = SP, so that if a panic happens
999 // during the function we are about to execute, it will
1000 // have a valid SP to run on the g0 stack.
1001 // The next few lines (after the havem label)
1002 // will save this SP onto the stack and then write
1003 // the same SP back to m->sched.sp. That seems redundant,
1004 // but if an unrecovered panic happens, unwindm will
1005 // restore the g->sched.sp from the stack location
1006 // and then systemstack will try to use it. If we don't set it here,
1007 // that restored SP will be uninitialized (typically 0) and
1008 // will not be usable.
1009 MOVD g_m(g), R8
1010 MOVD m_g0(R8), R3
1011 MOVD RSP, R0
1012 MOVD R0, (g_sched+gobuf_sp)(R3)
1013 MOVD R29, (g_sched+gobuf_bp)(R3)
1014
1015 havem:
1016 // Now there's a valid m, and we're running on its m->g0.
1017 // Save current m->g0->sched.sp on stack and then set it to SP.
1018 // Save current sp in m->g0->sched.sp in preparation for
1019 // switch back to m->curg stack.
1020 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
1021 // Beware that the frame size is actually 32+16.
1022 MOVD m_g0(R8), R3
1023 MOVD (g_sched+gobuf_sp)(R3), R4
1024 MOVD R4, savedsp-16(SP)
1025 MOVD RSP, R0
1026 MOVD R0, (g_sched+gobuf_sp)(R3)
1027
1028 // Switch to m->curg stack and call runtime.cgocallbackg.
1029 // Because we are taking over the execution of m->curg
1030 // but *not* resuming what had been running, we need to
1031 // save that information (m->curg->sched) so we can restore it.
1032 // We can restore m->curg->sched.sp easily, because calling
1033 // runtime.cgocallbackg leaves SP unchanged upon return.
1034 // To save m->curg->sched.pc, we push it onto the curg stack and
1035 // open a frame the same size as cgocallback's g0 frame.
1036 // Once we switch to the curg stack, the pushed PC will appear
1037 // to be the return PC of cgocallback, so that the traceback
1038 // will seamlessly trace back into the earlier calls.
1039 MOVD m_curg(R8), g
1040 BL runtime·save_g(SB)
1041 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
1042 MOVD (g_sched+gobuf_pc)(g), R5
1043 MOVD R5, -48(R4)
1044 MOVD (g_sched+gobuf_bp)(g), R5
1045 MOVD R5, -56(R4)
1046 // Gather our arguments into registers.
1047 MOVD fn+0(FP), R1
1048 MOVD frame+8(FP), R2
1049 MOVD ctxt+16(FP), R3
1050 MOVD $-48(R4), R0 // maintain 16-byte SP alignment
1051 MOVD R0, RSP // switch stack
1052 MOVD R1, 8(RSP)
1053 MOVD R2, 16(RSP)
1054 MOVD R3, 24(RSP)
1055 BL runtime·cgocallbackg(SB)
1056
1057 // Restore g->sched (== m->curg->sched) from saved values.
1058 MOVD 0(RSP), R5
1059 MOVD R5, (g_sched+gobuf_pc)(g)
1060 MOVD RSP, R4
1061 ADD $48, R4, R4
1062 MOVD R4, (g_sched+gobuf_sp)(g)
1063
1064 // Switch back to m->g0's stack and restore m->g0->sched.sp.
1065 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
1066 // so we do not have to restore it.)
1067 MOVD g_m(g), R8
1068 MOVD m_g0(R8), g
1069 BL runtime·save_g(SB)
1070 MOVD (g_sched+gobuf_sp)(g), R0
1071 MOVD R0, RSP
1072 MOVD savedsp-16(SP), R4
1073 MOVD R4, (g_sched+gobuf_sp)(g)
1074
1075 // If the m on entry was nil, we called needm above to borrow an m
1076 // for the duration of the call. Since the call is over, return it with dropm.
1077 MOVD savedm-8(SP), R6
1078 CBNZ R6, droppedm
1079 MOVD $runtime·dropm(SB), R0
1080 BL (R0)
1081 droppedm:
1082
1083 // Done!
1084 RET
1085
1086 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1087 // Must obey the gcc calling convention.
1088 TEXT _cgo_topofstack(SB),NOSPLIT,$24
1089 // g (R28) and REGTMP (R27) might be clobbered by load_g. They
1090 // are callee-save in the gcc calling convention, so save them.
1091 MOVD R27, savedR27-8(SP)
1092 MOVD g, saveG-16(SP)
1093
1094 BL runtime·load_g(SB)
1095 MOVD g_m(g), R0
1096 MOVD m_curg(R0), R0
1097 MOVD (g_stack+stack_hi)(R0), R0
1098
1099 MOVD saveG-16(SP), g
1100 MOVD savedR28-8(SP), R27
1101 RET
1102
1103 // void setg(G*); set g. for use by needm.
1104 TEXT runtime·setg(SB), NOSPLIT, $0-8
1105 MOVD gg+0(FP), g
1106 // This only happens if iscgo, so jump straight to save_g
1107 BL runtime·save_g(SB)
1108 RET
1109
1110 // void setg_gcc(G*); set g called from gcc
1111 TEXT setg_gcc<>(SB),NOSPLIT,$8
1112 MOVD R0, g
1113 MOVD R27, savedR27-8(SP)
1114 BL runtime·save_g(SB)
1115 MOVD savedR27-8(SP), R27
1116 RET
1117
1118 TEXT runtime·emptyfunc(SB),0,$0-0
1119 RET
1120
1121 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
1122 MOVD ZR, R0
1123 MOVD (R0), R0
1124 UNDEF
1125
1126 TEXT runtime·return0(SB), NOSPLIT, $0
1127 MOVW $0, R0
1128 RET
1129
1130 // The top-most function running on a goroutine
1131 // returns to goexit+PCQuantum.
1132 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1133 MOVD R0, R0 // NOP
1134 BL runtime·goexit1(SB) // does not return
1135
1136 // This is called from .init_array and follows the platform, not Go, ABI.
1137 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
1138 SUB $0x10, RSP
1139 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
1140 MOVD runtime·lastmoduledatap(SB), R1
1141 MOVD R0, moduledata_next(R1)
1142 MOVD R0, runtime·lastmoduledatap(SB)
1143 MOVD 8(RSP), R27
1144 ADD $0x10, RSP
1145 RET
1146
1147 TEXT ·checkASM(SB),NOSPLIT,$0-1
1148 MOVW $1, R3
1149 MOVB R3, ret+0(FP)
1150 RET
1151
1152 // gcWriteBarrier performs a heap pointer write and informs the GC.
1153 //
1154 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
1155 // - R2 is the destination of the write
1156 // - R3 is the value being written at R2
1157 // It clobbers condition codes.
1158 // It does not clobber any general-purpose registers,
1159 // but may clobber others (e.g., floating point registers)
1160 // The act of CALLing gcWriteBarrier will clobber R30 (LR).
1161 TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$200
1162 // Save the registers clobbered by the fast path.
1163 MOVD R0, 184(RSP)
1164 MOVD R1, 192(RSP)
1165 MOVD g_m(g), R0
1166 MOVD m_p(R0), R0
1167 MOVD (p_wbBuf+wbBuf_next)(R0), R1
1168 // Increment wbBuf.next position.
1169 ADD $16, R1
1170 MOVD R1, (p_wbBuf+wbBuf_next)(R0)
1171 MOVD (p_wbBuf+wbBuf_end)(R0), R0
1172 CMP R1, R0
1173 // Record the write.
1174 MOVD R3, -16(R1) // Record value
1175 MOVD (R2), R0 // TODO: This turns bad writes into bad reads.
1176 MOVD R0, -8(R1) // Record *slot
1177 // Is the buffer full? (flags set in CMP above)
1178 BEQ flush
1179 ret:
1180 MOVD 184(RSP), R0
1181 MOVD 192(RSP), R1
1182 // Do the write.
1183 MOVD R3, (R2)
1184 RET
1185
1186 flush:
1187 // Save all general purpose registers since these could be
1188 // clobbered by wbBufFlush and were not saved by the caller.
1189 MOVD R2, 8(RSP) // Also first argument to wbBufFlush
1190 MOVD R3, 16(RSP) // Also second argument to wbBufFlush
1191 // R0 already saved
1192 // R1 already saved
1193 MOVD R4, 24(RSP)
1194 MOVD R5, 32(RSP)
1195 MOVD R6, 40(RSP)
1196 MOVD R7, 48(RSP)
1197 MOVD R8, 56(RSP)
1198 MOVD R9, 64(RSP)
1199 MOVD R10, 72(RSP)
1200 MOVD R11, 80(RSP)
1201 MOVD R12, 88(RSP)
1202 MOVD R13, 96(RSP)
1203 MOVD R14, 104(RSP)
1204 MOVD R15, 112(RSP)
1205 // R16, R17 may be clobbered by linker trampoline
1206 // R18 is unused.
1207 MOVD R19, 120(RSP)
1208 MOVD R20, 128(RSP)
1209 MOVD R21, 136(RSP)
1210 MOVD R22, 144(RSP)
1211 MOVD R23, 152(RSP)
1212 MOVD R24, 160(RSP)
1213 MOVD R25, 168(RSP)
1214 MOVD R26, 176(RSP)
1215 // R27 is temp register.
1216 // R28 is g.
1217 // R29 is frame pointer (unused).
1218 // R30 is LR, which was saved by the prologue.
1219 // R31 is SP.
1220
1221 // This takes arguments R2 and R3.
1222 CALL runtime·wbBufFlush(SB)
1223
1224 MOVD 8(RSP), R2
1225 MOVD 16(RSP), R3
1226 MOVD 24(RSP), R4
1227 MOVD 32(RSP), R5
1228 MOVD 40(RSP), R6
1229 MOVD 48(RSP), R7
1230 MOVD 56(RSP), R8
1231 MOVD 64(RSP), R9
1232 MOVD 72(RSP), R10
1233 MOVD 80(RSP), R11
1234 MOVD 88(RSP), R12
1235 MOVD 96(RSP), R13
1236 MOVD 104(RSP), R14
1237 MOVD 112(RSP), R15
1238 MOVD 120(RSP), R19
1239 MOVD 128(RSP), R20
1240 MOVD 136(RSP), R21
1241 MOVD 144(RSP), R22
1242 MOVD 152(RSP), R23
1243 MOVD 160(RSP), R24
1244 MOVD 168(RSP), R25
1245 MOVD 176(RSP), R26
1246 JMP ret
1247
1248 // Note: these functions use a special calling convention to save generated code space.
1249 // Arguments are passed in registers, but the space for those arguments are allocated
1250 // in the caller's stack frame. These stubs write the args into that stack space and
1251 // then tail call to the corresponding runtime handler.
1252 // The tail call makes these stubs disappear in backtraces.
1253 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
1254 MOVD R0, x+0(FP)
1255 MOVD R1, y+8(FP)
1256 JMP runtime·goPanicIndex(SB)
1257 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
1258 MOVD R0, x+0(FP)
1259 MOVD R1, y+8(FP)
1260 JMP runtime·goPanicIndexU(SB)
1261 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
1262 MOVD R1, x+0(FP)
1263 MOVD R2, y+8(FP)
1264 JMP runtime·goPanicSliceAlen(SB)
1265 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
1266 MOVD R1, x+0(FP)
1267 MOVD R2, y+8(FP)
1268 JMP runtime·goPanicSliceAlenU(SB)
1269 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
1270 MOVD R1, x+0(FP)
1271 MOVD R2, y+8(FP)
1272 JMP runtime·goPanicSliceAcap(SB)
1273 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
1274 MOVD R1, x+0(FP)
1275 MOVD R2, y+8(FP)
1276 JMP runtime·goPanicSliceAcapU(SB)
1277 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
1278 MOVD R0, x+0(FP)
1279 MOVD R1, y+8(FP)
1280 JMP runtime·goPanicSliceB(SB)
1281 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
1282 MOVD R0, x+0(FP)
1283 MOVD R1, y+8(FP)
1284 JMP runtime·goPanicSliceBU(SB)
1285 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
1286 MOVD R2, x+0(FP)
1287 MOVD R3, y+8(FP)
1288 JMP runtime·goPanicSlice3Alen(SB)
1289 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
1290 MOVD R2, x+0(FP)
1291 MOVD R3, y+8(FP)
1292 JMP runtime·goPanicSlice3AlenU(SB)
1293 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
1294 MOVD R2, x+0(FP)
1295 MOVD R3, y+8(FP)
1296 JMP runtime·goPanicSlice3Acap(SB)
1297 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
1298 MOVD R2, x+0(FP)
1299 MOVD R3, y+8(FP)
1300 JMP runtime·goPanicSlice3AcapU(SB)
1301 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
1302 MOVD R1, x+0(FP)
1303 MOVD R2, y+8(FP)
1304 JMP runtime·goPanicSlice3B(SB)
1305 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
1306 MOVD R1, x+0(FP)
1307 MOVD R2, y+8(FP)
1308 JMP runtime·goPanicSlice3BU(SB)
1309 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
1310 MOVD R0, x+0(FP)
1311 MOVD R1, y+8(FP)
1312 JMP runtime·goPanicSlice3C(SB)
1313 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
1314 MOVD R0, x+0(FP)
1315 MOVD R1, y+8(FP)
1316 JMP runtime·goPanicSlice3CU(SB)
1317 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
1318 MOVD R2, x+0(FP)
1319 MOVD R3, y+8(FP)
1320 JMP runtime·goPanicSliceConvert(SB)
1321
View as plain text