Text file
src/runtime/asm_arm.s
Documentation: runtime
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_arm is common startup code for most ARM systems when using
11 // internal linking. This is the entry point for the program from the
12 // kernel for an ordinary -buildmode=exe program. The stack holds the
13 // number of arguments and the C-style argv.
14 TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0
15 MOVW (R13), R0 // argc
16 MOVW $4(R13), R1 // argv
17 B runtime·rt0_go(SB)
18
19 // main is common startup code for most ARM systems when using
20 // external linking. The C startup code will call the symbol "main"
21 // passing argc and argv in the usual C ABI registers R0 and R1.
22 TEXT main(SB),NOSPLIT|NOFRAME,$0
23 B runtime·rt0_go(SB)
24
25 // _rt0_arm_lib is common startup code for most ARM systems when
26 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
27 // arrange to invoke this function as a global constructor (for
28 // c-archive) or when the shared library is loaded (for c-shared).
29 // We expect argc and argv to be passed in the usual C ABI registers
30 // R0 and R1.
31 TEXT _rt0_arm_lib(SB),NOSPLIT,$104
32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example,
33 // actually cares that R11 is preserved.
34 MOVW R4, 12(R13)
35 MOVW R5, 16(R13)
36 MOVW R6, 20(R13)
37 MOVW R7, 24(R13)
38 MOVW R8, 28(R13)
39 MOVW g, 32(R13)
40 MOVW R11, 36(R13)
41
42 // Skip floating point registers on GOARM < 6.
43 MOVB runtime·goarm(SB), R11
44 CMP $6, R11
45 BLT skipfpsave
46 MOVD F8, (40+8*0)(R13)
47 MOVD F9, (40+8*1)(R13)
48 MOVD F10, (40+8*2)(R13)
49 MOVD F11, (40+8*3)(R13)
50 MOVD F12, (40+8*4)(R13)
51 MOVD F13, (40+8*5)(R13)
52 MOVD F14, (40+8*6)(R13)
53 MOVD F15, (40+8*7)(R13)
54 skipfpsave:
55 // Save argc/argv.
56 MOVW R0, _rt0_arm_lib_argc<>(SB)
57 MOVW R1, _rt0_arm_lib_argv<>(SB)
58
59 MOVW $0, g // Initialize g.
60
61 // Synchronous initialization.
62 CALL runtime·libpreinit(SB)
63
64 // Create a new thread to do the runtime initialization.
65 MOVW _cgo_sys_thread_create(SB), R2
66 CMP $0, R2
67 BEQ nocgo
68 MOVW $_rt0_arm_lib_go<>(SB), R0
69 MOVW $0, R1
70 BL (R2)
71 B rr
72 nocgo:
73 MOVW $0x800000, R0 // stacksize = 8192KB
74 MOVW $_rt0_arm_lib_go<>(SB), R1 // fn
75 MOVW R0, 4(R13)
76 MOVW R1, 8(R13)
77 BL runtime·newosproc0(SB)
78 rr:
79 // Restore callee-save registers and return.
80 MOVB runtime·goarm(SB), R11
81 CMP $6, R11
82 BLT skipfprest
83 MOVD (40+8*0)(R13), F8
84 MOVD (40+8*1)(R13), F9
85 MOVD (40+8*2)(R13), F10
86 MOVD (40+8*3)(R13), F11
87 MOVD (40+8*4)(R13), F12
88 MOVD (40+8*5)(R13), F13
89 MOVD (40+8*6)(R13), F14
90 MOVD (40+8*7)(R13), F15
91 skipfprest:
92 MOVW 12(R13), R4
93 MOVW 16(R13), R5
94 MOVW 20(R13), R6
95 MOVW 24(R13), R7
96 MOVW 28(R13), R8
97 MOVW 32(R13), g
98 MOVW 36(R13), R11
99 RET
100
101 // _rt0_arm_lib_go initializes the Go runtime.
102 // This is started in a separate thread by _rt0_arm_lib.
103 TEXT _rt0_arm_lib_go<>(SB),NOSPLIT,$8
104 MOVW _rt0_arm_lib_argc<>(SB), R0
105 MOVW _rt0_arm_lib_argv<>(SB), R1
106 B runtime·rt0_go(SB)
107
108 DATA _rt0_arm_lib_argc<>(SB)/4,$0
109 GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4
110 DATA _rt0_arm_lib_argv<>(SB)/4,$0
111 GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4
112
113 // using NOFRAME means do not save LR on stack.
114 // argc is in R0, argv is in R1.
115 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
116 MOVW $0xcafebabe, R12
117
118 // copy arguments forward on an even stack
119 // use R13 instead of SP to avoid linker rewriting the offsets
120 SUB $64, R13 // plenty of scratch
121 AND $~7, R13
122 MOVW R0, 60(R13) // save argc, argv away
123 MOVW R1, 64(R13)
124
125 // set up g register
126 // g is R10
127 MOVW $runtime·g0(SB), g
128 MOVW $runtime·m0(SB), R8
129
130 // save m->g0 = g0
131 MOVW g, m_g0(R8)
132 // save g->m = m0
133 MOVW R8, g_m(g)
134
135 // create istack out of the OS stack
136 // (1MB of system stack is available on iOS and Android)
137 MOVW $(-64*1024+104)(R13), R0
138 MOVW R0, g_stackguard0(g)
139 MOVW R0, g_stackguard1(g)
140 MOVW R0, (g_stack+stack_lo)(g)
141 MOVW R13, (g_stack+stack_hi)(g)
142
143 BL runtime·emptyfunc(SB) // fault if stack check is wrong
144
145 #ifdef GOOS_openbsd
146 // Save g to TLS so that it is available from signal trampoline.
147 BL runtime·save_g(SB)
148 #endif
149
150 BL runtime·_initcgo(SB) // will clobber R0-R3
151
152 // update stackguard after _cgo_init
153 MOVW (g_stack+stack_lo)(g), R0
154 ADD $const__StackGuard, R0
155 MOVW R0, g_stackguard0(g)
156 MOVW R0, g_stackguard1(g)
157
158 BL runtime·check(SB)
159
160 // saved argc, argv
161 MOVW 60(R13), R0
162 MOVW R0, 4(R13)
163 MOVW 64(R13), R1
164 MOVW R1, 8(R13)
165 BL runtime·args(SB)
166 BL runtime·checkgoarm(SB)
167 BL runtime·osinit(SB)
168 BL runtime·schedinit(SB)
169
170 // create a new goroutine to start program
171 MOVW $runtime·mainPC(SB), R0
172 MOVW.W R0, -4(R13)
173 MOVW $8, R0
174 MOVW.W R0, -4(R13)
175 MOVW $0, R0
176 MOVW.W R0, -4(R13) // push $0 as guard
177 BL runtime·newproc(SB)
178 MOVW $12(R13), R13 // pop args and LR
179
180 // start this M
181 BL runtime·mstart(SB)
182
183 MOVW $1234, R0
184 MOVW $1000, R1
185 MOVW R0, (R1) // fail hard
186
187 DATA runtime·mainPC+0(SB)/4,$runtime·main(SB)
188 GLOBL runtime·mainPC(SB),RODATA,$4
189
190 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
191 // gdb won't skip this breakpoint instruction automatically,
192 // so you must manually "set $pc+=4" to skip it and continue.
193 #ifdef GOOS_plan9
194 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9
195 #else
196 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
197 #endif
198 RET
199
200 TEXT runtime·asminit(SB),NOSPLIT,$0-0
201 // disable runfast (flush-to-zero) mode of vfp if runtime.goarm > 5
202 MOVB runtime·goarm(SB), R11
203 CMP $5, R11
204 BLE 4(PC)
205 WORD $0xeef1ba10 // vmrs r11, fpscr
206 BIC $(1<<24), R11
207 WORD $0xeee1ba10 // vmsr fpscr, r11
208 RET
209
210 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
211 BL runtime·mstart0(SB)
212 RET // not reached
213
214 /*
215 * go-routine
216 */
217
218 // void gogo(Gobuf*)
219 // restore state from Gobuf; longjmp
220 TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4
221 MOVW buf+0(FP), R1
222 MOVW gobuf_g(R1), R0
223 MOVW 0(R0), R2 // make sure g != nil
224 B gogo<>(SB)
225
226 TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0
227 BL setg<>(SB)
228 MOVW gobuf_sp(R1), R13 // restore SP==R13
229 MOVW gobuf_lr(R1), LR
230 MOVW gobuf_ret(R1), R0
231 MOVW gobuf_ctxt(R1), R7
232 MOVW $0, R11
233 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
234 MOVW R11, gobuf_ret(R1)
235 MOVW R11, gobuf_lr(R1)
236 MOVW R11, gobuf_ctxt(R1)
237 MOVW gobuf_pc(R1), R11
238 CMP R11, R11 // set condition codes for == test, needed by stack split
239 B (R11)
240
241 // func mcall(fn func(*g))
242 // Switch to m->g0's stack, call fn(g).
243 // Fn must never return. It should gogo(&g->sched)
244 // to keep running g.
245 TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4
246 // Save caller state in g->sched.
247 MOVW R13, (g_sched+gobuf_sp)(g)
248 MOVW LR, (g_sched+gobuf_pc)(g)
249 MOVW $0, R11
250 MOVW R11, (g_sched+gobuf_lr)(g)
251
252 // Switch to m->g0 & its stack, call fn.
253 MOVW g, R1
254 MOVW g_m(g), R8
255 MOVW m_g0(R8), R0
256 BL setg<>(SB)
257 CMP g, R1
258 B.NE 2(PC)
259 B runtime·badmcall(SB)
260 MOVW fn+0(FP), R0
261 MOVW (g_sched+gobuf_sp)(g), R13
262 SUB $8, R13
263 MOVW R1, 4(R13)
264 MOVW R0, R7
265 MOVW 0(R0), R0
266 BL (R0)
267 B runtime·badmcall2(SB)
268 RET
269
270 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
271 // of the G stack. We need to distinguish the routine that
272 // lives at the bottom of the G stack from the one that lives
273 // at the top of the system stack because the one at the top of
274 // the system stack terminates the stack walk (see topofstack()).
275 TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0
276 MOVW $0, R0
277 BL (R0) // clobber lr to ensure push {lr} is kept
278 RET
279
280 // func systemstack(fn func())
281 TEXT runtime·systemstack(SB),NOSPLIT,$0-4
282 MOVW fn+0(FP), R0 // R0 = fn
283 MOVW g_m(g), R1 // R1 = m
284
285 MOVW m_gsignal(R1), R2 // R2 = gsignal
286 CMP g, R2
287 B.EQ noswitch
288
289 MOVW m_g0(R1), R2 // R2 = g0
290 CMP g, R2
291 B.EQ noswitch
292
293 MOVW m_curg(R1), R3
294 CMP g, R3
295 B.EQ switch
296
297 // Bad: g is not gsignal, not g0, not curg. What is it?
298 // Hide call from linker nosplit analysis.
299 MOVW $runtime·badsystemstack(SB), R0
300 BL (R0)
301 B runtime·abort(SB)
302
303 switch:
304 // save our state in g->sched. Pretend to
305 // be systemstack_switch if the G stack is scanned.
306 BL gosave_systemstack_switch<>(SB)
307
308 // switch to g0
309 MOVW R0, R5
310 MOVW R2, R0
311 BL setg<>(SB)
312 MOVW R5, R0
313 MOVW (g_sched+gobuf_sp)(R2), R13
314
315 // call target function
316 MOVW R0, R7
317 MOVW 0(R0), R0
318 BL (R0)
319
320 // switch back to g
321 MOVW g_m(g), R1
322 MOVW m_curg(R1), R0
323 BL setg<>(SB)
324 MOVW (g_sched+gobuf_sp)(g), R13
325 MOVW $0, R3
326 MOVW R3, (g_sched+gobuf_sp)(g)
327 RET
328
329 noswitch:
330 // Using a tail call here cleans up tracebacks since we won't stop
331 // at an intermediate systemstack.
332 MOVW R0, R7
333 MOVW 0(R0), R0
334 MOVW.P 4(R13), R14 // restore LR
335 B (R0)
336
337 /*
338 * support for morestack
339 */
340
341 // Called during function prolog when more stack is needed.
342 // R3 prolog's LR
343 // using NOFRAME means do not save LR on stack.
344 //
345 // The traceback routines see morestack on a g0 as being
346 // the top of a stack (for example, morestack calling newstack
347 // calling the scheduler calling newm calling gc), so we must
348 // record an argument size. For that purpose, it has no arguments.
349 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
350 // Cannot grow scheduler stack (m->g0).
351 MOVW g_m(g), R8
352 MOVW m_g0(R8), R4
353 CMP g, R4
354 BNE 3(PC)
355 BL runtime·badmorestackg0(SB)
356 B runtime·abort(SB)
357
358 // Cannot grow signal stack (m->gsignal).
359 MOVW m_gsignal(R8), R4
360 CMP g, R4
361 BNE 3(PC)
362 BL runtime·badmorestackgsignal(SB)
363 B runtime·abort(SB)
364
365 // Called from f.
366 // Set g->sched to context in f.
367 MOVW R13, (g_sched+gobuf_sp)(g)
368 MOVW LR, (g_sched+gobuf_pc)(g)
369 MOVW R3, (g_sched+gobuf_lr)(g)
370 MOVW R7, (g_sched+gobuf_ctxt)(g)
371
372 // Called from f.
373 // Set m->morebuf to f's caller.
374 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
375 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
376 MOVW g, (m_morebuf+gobuf_g)(R8)
377
378 // Call newstack on m->g0's stack.
379 MOVW m_g0(R8), R0
380 BL setg<>(SB)
381 MOVW (g_sched+gobuf_sp)(g), R13
382 MOVW $0, R0
383 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR)
384 BL runtime·newstack(SB)
385
386 // Not reached, but make sure the return PC from the call to newstack
387 // is still in this function, and not the beginning of the next.
388 RET
389
390 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
391 MOVW $0, R7
392 B runtime·morestack(SB)
393
394 // reflectcall: call a function with the given argument list
395 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
396 // we don't have variable-sized frames, so we use a small number
397 // of constant-sized-frame functions to encode a few bits of size in the pc.
398 // Caution: ugly multiline assembly macros in your future!
399
400 #define DISPATCH(NAME,MAXSIZE) \
401 CMP $MAXSIZE, R0; \
402 B.HI 3(PC); \
403 MOVW $NAME(SB), R1; \
404 B (R1)
405
406 TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
407 MOVW frameSize+20(FP), R0
408 DISPATCH(runtime·call16, 16)
409 DISPATCH(runtime·call32, 32)
410 DISPATCH(runtime·call64, 64)
411 DISPATCH(runtime·call128, 128)
412 DISPATCH(runtime·call256, 256)
413 DISPATCH(runtime·call512, 512)
414 DISPATCH(runtime·call1024, 1024)
415 DISPATCH(runtime·call2048, 2048)
416 DISPATCH(runtime·call4096, 4096)
417 DISPATCH(runtime·call8192, 8192)
418 DISPATCH(runtime·call16384, 16384)
419 DISPATCH(runtime·call32768, 32768)
420 DISPATCH(runtime·call65536, 65536)
421 DISPATCH(runtime·call131072, 131072)
422 DISPATCH(runtime·call262144, 262144)
423 DISPATCH(runtime·call524288, 524288)
424 DISPATCH(runtime·call1048576, 1048576)
425 DISPATCH(runtime·call2097152, 2097152)
426 DISPATCH(runtime·call4194304, 4194304)
427 DISPATCH(runtime·call8388608, 8388608)
428 DISPATCH(runtime·call16777216, 16777216)
429 DISPATCH(runtime·call33554432, 33554432)
430 DISPATCH(runtime·call67108864, 67108864)
431 DISPATCH(runtime·call134217728, 134217728)
432 DISPATCH(runtime·call268435456, 268435456)
433 DISPATCH(runtime·call536870912, 536870912)
434 DISPATCH(runtime·call1073741824, 1073741824)
435 MOVW $runtime·badreflectcall(SB), R1
436 B (R1)
437
438 #define CALLFN(NAME,MAXSIZE) \
439 TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
440 NO_LOCAL_POINTERS; \
441 /* copy arguments to stack */ \
442 MOVW stackArgs+8(FP), R0; \
443 MOVW stackArgsSize+12(FP), R2; \
444 ADD $4, R13, R1; \
445 CMP $0, R2; \
446 B.EQ 5(PC); \
447 MOVBU.P 1(R0), R5; \
448 MOVBU.P R5, 1(R1); \
449 SUB $1, R2, R2; \
450 B -5(PC); \
451 /* call function */ \
452 MOVW f+4(FP), R7; \
453 MOVW (R7), R0; \
454 PCDATA $PCDATA_StackMapIndex, $0; \
455 BL (R0); \
456 /* copy return values back */ \
457 MOVW stackArgsType+0(FP), R4; \
458 MOVW stackArgs+8(FP), R0; \
459 MOVW stackArgsSize+12(FP), R2; \
460 MOVW stackArgsRetOffset+16(FP), R3; \
461 ADD $4, R13, R1; \
462 ADD R3, R1; \
463 ADD R3, R0; \
464 SUB R3, R2; \
465 BL callRet<>(SB); \
466 RET
467
468 // callRet copies return values back at the end of call*. This is a
469 // separate function so it can allocate stack space for the arguments
470 // to reflectcallmove. It does not follow the Go ABI; it expects its
471 // arguments in registers.
472 TEXT callRet<>(SB), NOSPLIT, $20-0
473 MOVW R4, 4(R13)
474 MOVW R0, 8(R13)
475 MOVW R1, 12(R13)
476 MOVW R2, 16(R13)
477 MOVW $0, R7
478 MOVW R7, 20(R13)
479 BL runtime·reflectcallmove(SB)
480 RET
481
482 CALLFN(·call16, 16)
483 CALLFN(·call32, 32)
484 CALLFN(·call64, 64)
485 CALLFN(·call128, 128)
486 CALLFN(·call256, 256)
487 CALLFN(·call512, 512)
488 CALLFN(·call1024, 1024)
489 CALLFN(·call2048, 2048)
490 CALLFN(·call4096, 4096)
491 CALLFN(·call8192, 8192)
492 CALLFN(·call16384, 16384)
493 CALLFN(·call32768, 32768)
494 CALLFN(·call65536, 65536)
495 CALLFN(·call131072, 131072)
496 CALLFN(·call262144, 262144)
497 CALLFN(·call524288, 524288)
498 CALLFN(·call1048576, 1048576)
499 CALLFN(·call2097152, 2097152)
500 CALLFN(·call4194304, 4194304)
501 CALLFN(·call8388608, 8388608)
502 CALLFN(·call16777216, 16777216)
503 CALLFN(·call33554432, 33554432)
504 CALLFN(·call67108864, 67108864)
505 CALLFN(·call134217728, 134217728)
506 CALLFN(·call268435456, 268435456)
507 CALLFN(·call536870912, 536870912)
508 CALLFN(·call1073741824, 1073741824)
509
510 // void jmpdefer(fn, sp);
511 // called from deferreturn.
512 // 1. grab stored LR for caller
513 // 2. sub 4 bytes to get back to BL deferreturn
514 // 3. B to fn
515 TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8
516 MOVW 0(R13), LR
517 MOVW $-4(LR), LR // BL deferreturn
518 MOVW fv+0(FP), R7
519 MOVW argp+4(FP), R13
520 MOVW $-4(R13), R13 // SP is 4 below argp, due to saved LR
521 MOVW 0(R7), R1
522 B (R1)
523
524 // Save state of caller into g->sched,
525 // but using fake PC from systemstack_switch.
526 // Must only be called from functions with no locals ($0)
527 // or else unwinding from systemstack_switch is incorrect.
528 // Smashes R11.
529 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
530 MOVW $runtime·systemstack_switch(SB), R11
531 ADD $4, R11 // get past push {lr}
532 MOVW R11, (g_sched+gobuf_pc)(g)
533 MOVW R13, (g_sched+gobuf_sp)(g)
534 MOVW $0, R11
535 MOVW R11, (g_sched+gobuf_lr)(g)
536 MOVW R11, (g_sched+gobuf_ret)(g)
537 // Assert ctxt is zero. See func save.
538 MOVW (g_sched+gobuf_ctxt)(g), R11
539 TST R11, R11
540 B.EQ 2(PC)
541 BL runtime·abort(SB)
542 RET
543
544 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
545 // Call fn(arg) aligned appropriately for the gcc ABI.
546 // Called on a system stack, and there may be no g yet (during needm).
547 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8
548 MOVW fn+0(FP), R1
549 MOVW arg+4(FP), R0
550 MOVW R13, R2
551 SUB $32, R13
552 BIC $0x7, R13 // alignment for gcc ABI
553 MOVW R2, 8(R13)
554 BL (R1)
555 MOVW 8(R13), R2
556 MOVW R2, R13
557 RET
558
559 // func asmcgocall(fn, arg unsafe.Pointer) int32
560 // Call fn(arg) on the scheduler stack,
561 // aligned appropriately for the gcc ABI.
562 // See cgocall.go for more details.
563 TEXT ·asmcgocall(SB),NOSPLIT,$0-12
564 MOVW fn+0(FP), R1
565 MOVW arg+4(FP), R0
566
567 MOVW R13, R2
568 CMP $0, g
569 BEQ nosave
570 MOVW g, R4
571
572 // Figure out if we need to switch to m->g0 stack.
573 // We get called to create new OS threads too, and those
574 // come in on the m->g0 stack already.
575 MOVW g_m(g), R8
576 MOVW m_gsignal(R8), R3
577 CMP R3, g
578 BEQ nosave
579 MOVW m_g0(R8), R3
580 CMP R3, g
581 BEQ nosave
582 BL gosave_systemstack_switch<>(SB)
583 MOVW R0, R5
584 MOVW R3, R0
585 BL setg<>(SB)
586 MOVW R5, R0
587 MOVW (g_sched+gobuf_sp)(g), R13
588
589 // Now on a scheduling stack (a pthread-created stack).
590 SUB $24, R13
591 BIC $0x7, R13 // alignment for gcc ABI
592 MOVW R4, 20(R13) // save old g
593 MOVW (g_stack+stack_hi)(R4), R4
594 SUB R2, R4
595 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
596 BL (R1)
597
598 // Restore registers, g, stack pointer.
599 MOVW R0, R5
600 MOVW 20(R13), R0
601 BL setg<>(SB)
602 MOVW (g_stack+stack_hi)(g), R1
603 MOVW 16(R13), R2
604 SUB R2, R1
605 MOVW R5, R0
606 MOVW R1, R13
607
608 MOVW R0, ret+8(FP)
609 RET
610
611 nosave:
612 // Running on a system stack, perhaps even without a g.
613 // Having no g can happen during thread creation or thread teardown
614 // (see needm/dropm on Solaris, for example).
615 // This code is like the above sequence but without saving/restoring g
616 // and without worrying about the stack moving out from under us
617 // (because we're on a system stack, not a goroutine stack).
618 // The above code could be used directly if already on a system stack,
619 // but then the only path through this code would be a rare case on Solaris.
620 // Using this code for all "already on system stack" calls exercises it more,
621 // which should help keep it correct.
622 SUB $24, R13
623 BIC $0x7, R13 // alignment for gcc ABI
624 // save null g in case someone looks during debugging.
625 MOVW $0, R4
626 MOVW R4, 20(R13)
627 MOVW R2, 16(R13) // Save old stack pointer.
628 BL (R1)
629 // Restore stack pointer.
630 MOVW 16(R13), R2
631 MOVW R2, R13
632 MOVW R0, ret+8(FP)
633 RET
634
635 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
636 // See cgocall.go for more details.
637 TEXT ·cgocallback(SB),NOSPLIT,$12-12
638 NO_LOCAL_POINTERS
639
640 // Load m and g from thread-local storage.
641 #ifdef GOOS_openbsd
642 BL runtime·load_g(SB)
643 #else
644 MOVB runtime·iscgo(SB), R0
645 CMP $0, R0
646 BL.NE runtime·load_g(SB)
647 #endif
648
649 // If g is nil, Go did not create the current thread.
650 // Call needm to obtain one for temporary use.
651 // In this case, we're running on the thread stack, so there's
652 // lots of space, but the linker doesn't know. Hide the call from
653 // the linker analysis by using an indirect call.
654 CMP $0, g
655 B.EQ needm
656
657 MOVW g_m(g), R8
658 MOVW R8, savedm-4(SP)
659 B havem
660
661 needm:
662 MOVW g, savedm-4(SP) // g is zero, so is m.
663 MOVW $runtime·needm(SB), R0
664 BL (R0)
665
666 // Set m->g0->sched.sp = SP, so that if a panic happens
667 // during the function we are about to execute, it will
668 // have a valid SP to run on the g0 stack.
669 // The next few lines (after the havem label)
670 // will save this SP onto the stack and then write
671 // the same SP back to m->sched.sp. That seems redundant,
672 // but if an unrecovered panic happens, unwindm will
673 // restore the g->sched.sp from the stack location
674 // and then systemstack will try to use it. If we don't set it here,
675 // that restored SP will be uninitialized (typically 0) and
676 // will not be usable.
677 MOVW g_m(g), R8
678 MOVW m_g0(R8), R3
679 MOVW R13, (g_sched+gobuf_sp)(R3)
680
681 havem:
682 // Now there's a valid m, and we're running on its m->g0.
683 // Save current m->g0->sched.sp on stack and then set it to SP.
684 // Save current sp in m->g0->sched.sp in preparation for
685 // switch back to m->curg stack.
686 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP).
687 MOVW m_g0(R8), R3
688 MOVW (g_sched+gobuf_sp)(R3), R4
689 MOVW R4, savedsp-12(SP) // must match frame size
690 MOVW R13, (g_sched+gobuf_sp)(R3)
691
692 // Switch to m->curg stack and call runtime.cgocallbackg.
693 // Because we are taking over the execution of m->curg
694 // but *not* resuming what had been running, we need to
695 // save that information (m->curg->sched) so we can restore it.
696 // We can restore m->curg->sched.sp easily, because calling
697 // runtime.cgocallbackg leaves SP unchanged upon return.
698 // To save m->curg->sched.pc, we push it onto the curg stack and
699 // open a frame the same size as cgocallback's g0 frame.
700 // Once we switch to the curg stack, the pushed PC will appear
701 // to be the return PC of cgocallback, so that the traceback
702 // will seamlessly trace back into the earlier calls.
703 MOVW m_curg(R8), R0
704 BL setg<>(SB)
705 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
706 MOVW (g_sched+gobuf_pc)(g), R5
707 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size
708 // Gather our arguments into registers.
709 MOVW fn+0(FP), R1
710 MOVW frame+4(FP), R2
711 MOVW ctxt+8(FP), R3
712 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size
713 MOVW R1, 4(R13)
714 MOVW R2, 8(R13)
715 MOVW R3, 12(R13)
716 BL runtime·cgocallbackg(SB)
717
718 // Restore g->sched (== m->curg->sched) from saved values.
719 MOVW 0(R13), R5
720 MOVW R5, (g_sched+gobuf_pc)(g)
721 MOVW $(12+4)(R13), R4 // must match frame size
722 MOVW R4, (g_sched+gobuf_sp)(g)
723
724 // Switch back to m->g0's stack and restore m->g0->sched.sp.
725 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
726 // so we do not have to restore it.)
727 MOVW g_m(g), R8
728 MOVW m_g0(R8), R0
729 BL setg<>(SB)
730 MOVW (g_sched+gobuf_sp)(g), R13
731 MOVW savedsp-12(SP), R4 // must match frame size
732 MOVW R4, (g_sched+gobuf_sp)(g)
733
734 // If the m on entry was nil, we called needm above to borrow an m
735 // for the duration of the call. Since the call is over, return it with dropm.
736 MOVW savedm-4(SP), R6
737 CMP $0, R6
738 B.NE 3(PC)
739 MOVW $runtime·dropm(SB), R0
740 BL (R0)
741
742 // Done!
743 RET
744
745 // void setg(G*); set g. for use by needm.
746 TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4
747 MOVW gg+0(FP), R0
748 B setg<>(SB)
749
750 TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0
751 MOVW R0, g
752
753 // Save g to thread-local storage.
754 #ifdef GOOS_windows
755 B runtime·save_g(SB)
756 #else
757 #ifdef GOOS_openbsd
758 B runtime·save_g(SB)
759 #else
760 MOVB runtime·iscgo(SB), R0
761 CMP $0, R0
762 B.EQ 2(PC)
763 B runtime·save_g(SB)
764
765 MOVW g, R0
766 RET
767 #endif
768 #endif
769
770 TEXT runtime·emptyfunc(SB),0,$0-0
771 RET
772
773 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
774 MOVW $0, R0
775 MOVW (R0), R1
776
777 // armPublicationBarrier is a native store/store barrier for ARMv7+.
778 // On earlier ARM revisions, armPublicationBarrier is a no-op.
779 // This will not work on SMP ARMv6 machines, if any are in use.
780 // To implement publicationBarrier in sys_$GOOS_arm.s using the native
781 // instructions, use:
782 //
783 // TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
784 // B runtime·armPublicationBarrier(SB)
785 //
786 TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
787 MOVB runtime·goarm(SB), R11
788 CMP $7, R11
789 BLT 2(PC)
790 DMB MB_ST
791 RET
792
793 // AES hashing not implemented for ARM
794 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16
795 JMP runtime·memhashFallback(SB)
796 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12
797 JMP runtime·strhashFallback(SB)
798 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12
799 JMP runtime·memhash32Fallback(SB)
800 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12
801 JMP runtime·memhash64Fallback(SB)
802
803 TEXT runtime·return0(SB),NOSPLIT,$0
804 MOVW $0, R0
805 RET
806
807 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0
808 MOVW cycles+0(FP), R1
809 MOVW $0, R0
810 yieldloop:
811 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K)
812 CMP R0, R1
813 B.NE 2(PC)
814 RET
815 SUB $1, R1
816 B yieldloop
817
818 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
819 // Must obey the gcc calling convention.
820 TEXT _cgo_topofstack(SB),NOSPLIT,$8
821 // R11 and g register are clobbered by load_g. They are
822 // callee-save in the gcc calling convention, so save them here.
823 MOVW R11, saveR11-4(SP)
824 MOVW g, saveG-8(SP)
825
826 BL runtime·load_g(SB)
827 MOVW g_m(g), R0
828 MOVW m_curg(R0), R0
829 MOVW (g_stack+stack_hi)(R0), R0
830
831 MOVW saveG-8(SP), g
832 MOVW saveR11-4(SP), R11
833 RET
834
835 // The top-most function running on a goroutine
836 // returns to goexit+PCQuantum.
837 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
838 MOVW R0, R0 // NOP
839 BL runtime·goexit1(SB) // does not return
840 // traceback from goexit1 must hit code range of goexit
841 MOVW R0, R0 // NOP
842
843 // x -> x/1000000, x%1000000, called from Go with args, results on stack.
844 TEXT runtime·usplit(SB),NOSPLIT,$0-12
845 MOVW x+0(FP), R0
846 CALL runtime·usplitR0(SB)
847 MOVW R0, q+4(FP)
848 MOVW R1, r+8(FP)
849 RET
850
851 // R0, R1 = R0/1000000, R0%1000000
852 TEXT runtime·usplitR0(SB),NOSPLIT,$0
853 // magic multiply to avoid software divide without available m.
854 // see output of go tool compile -S for x/1000000.
855 MOVW R0, R3
856 MOVW $1125899907, R1
857 MULLU R1, R0, (R0, R1)
858 MOVW R0>>18, R0
859 MOVW $1000000, R1
860 MULU R0, R1
861 SUB R1, R3, R1
862 RET
863
864 // This is called from .init_array and follows the platform, not Go, ABI.
865 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
866 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save
867 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI
868 MOVW runtime·lastmoduledatap(SB), R1
869 MOVW R0, moduledata_next(R1)
870 MOVW R0, runtime·lastmoduledatap(SB)
871 MOVW saver11-8(SP), R11
872 MOVW saver9-4(SP), R9
873 RET
874
875 TEXT ·checkASM(SB),NOSPLIT,$0-1
876 MOVW $1, R3
877 MOVB R3, ret+0(FP)
878 RET
879
880 // gcWriteBarrier performs a heap pointer write and informs the GC.
881 //
882 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
883 // - R2 is the destination of the write
884 // - R3 is the value being written at R2
885 // It clobbers condition codes.
886 // It does not clobber any other general-purpose registers,
887 // but may clobber others (e.g., floating point registers).
888 // The act of CALLing gcWriteBarrier will clobber R14 (LR).
889 TEXT runtime·gcWriteBarrier(SB),NOSPLIT|NOFRAME,$0
890 // Save the registers clobbered by the fast path.
891 MOVM.DB.W [R0,R1], (R13)
892 MOVW g_m(g), R0
893 MOVW m_p(R0), R0
894 MOVW (p_wbBuf+wbBuf_next)(R0), R1
895 // Increment wbBuf.next position.
896 ADD $8, R1
897 MOVW R1, (p_wbBuf+wbBuf_next)(R0)
898 MOVW (p_wbBuf+wbBuf_end)(R0), R0
899 CMP R1, R0
900 // Record the write.
901 MOVW R3, -8(R1) // Record value
902 MOVW (R2), R0 // TODO: This turns bad writes into bad reads.
903 MOVW R0, -4(R1) // Record *slot
904 // Is the buffer full? (flags set in CMP above)
905 B.EQ flush
906 ret:
907 MOVM.IA.W (R13), [R0,R1]
908 // Do the write.
909 MOVW R3, (R2)
910 RET
911
912 flush:
913 // Save all general purpose registers since these could be
914 // clobbered by wbBufFlush and were not saved by the caller.
915 //
916 // R0 and R1 were saved at entry.
917 // R10 is g, so preserved.
918 // R11 is linker temp, so no need to save.
919 // R13 is stack pointer.
920 // R15 is PC.
921 //
922 // This also sets up R2 and R3 as the arguments to wbBufFlush.
923 MOVM.DB.W [R2-R9,R12], (R13)
924 // Save R14 (LR) because the fast path above doesn't save it,
925 // but needs it to RET. This is after the MOVM so it appears below
926 // the arguments in the stack frame.
927 MOVM.DB.W [R14], (R13)
928
929 // This takes arguments R2 and R3.
930 CALL runtime·wbBufFlush(SB)
931
932 MOVM.IA.W (R13), [R14]
933 MOVM.IA.W (R13), [R2-R9,R12]
934 JMP ret
935
936 // Note: these functions use a special calling convention to save generated code space.
937 // Arguments are passed in registers, but the space for those arguments are allocated
938 // in the caller's stack frame. These stubs write the args into that stack space and
939 // then tail call to the corresponding runtime handler.
940 // The tail call makes these stubs disappear in backtraces.
941 TEXT runtime·panicIndex(SB),NOSPLIT,$0-8
942 MOVW R0, x+0(FP)
943 MOVW R1, y+4(FP)
944 JMP runtime·goPanicIndex(SB)
945 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8
946 MOVW R0, x+0(FP)
947 MOVW R1, y+4(FP)
948 JMP runtime·goPanicIndexU(SB)
949 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8
950 MOVW R1, x+0(FP)
951 MOVW R2, y+4(FP)
952 JMP runtime·goPanicSliceAlen(SB)
953 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8
954 MOVW R1, x+0(FP)
955 MOVW R2, y+4(FP)
956 JMP runtime·goPanicSliceAlenU(SB)
957 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8
958 MOVW R1, x+0(FP)
959 MOVW R2, y+4(FP)
960 JMP runtime·goPanicSliceAcap(SB)
961 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8
962 MOVW R1, x+0(FP)
963 MOVW R2, y+4(FP)
964 JMP runtime·goPanicSliceAcapU(SB)
965 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8
966 MOVW R0, x+0(FP)
967 MOVW R1, y+4(FP)
968 JMP runtime·goPanicSliceB(SB)
969 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8
970 MOVW R0, x+0(FP)
971 MOVW R1, y+4(FP)
972 JMP runtime·goPanicSliceBU(SB)
973 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8
974 MOVW R2, x+0(FP)
975 MOVW R3, y+4(FP)
976 JMP runtime·goPanicSlice3Alen(SB)
977 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8
978 MOVW R2, x+0(FP)
979 MOVW R3, y+4(FP)
980 JMP runtime·goPanicSlice3AlenU(SB)
981 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8
982 MOVW R2, x+0(FP)
983 MOVW R3, y+4(FP)
984 JMP runtime·goPanicSlice3Acap(SB)
985 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8
986 MOVW R2, x+0(FP)
987 MOVW R3, y+4(FP)
988 JMP runtime·goPanicSlice3AcapU(SB)
989 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8
990 MOVW R1, x+0(FP)
991 MOVW R2, y+4(FP)
992 JMP runtime·goPanicSlice3B(SB)
993 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8
994 MOVW R1, x+0(FP)
995 MOVW R2, y+4(FP)
996 JMP runtime·goPanicSlice3BU(SB)
997 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8
998 MOVW R0, x+0(FP)
999 MOVW R1, y+4(FP)
1000 JMP runtime·goPanicSlice3C(SB)
1001 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8
1002 MOVW R0, x+0(FP)
1003 MOVW R1, y+4(FP)
1004 JMP runtime·goPanicSlice3CU(SB)
1005 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8
1006 MOVW R2, x+0(FP)
1007 MOVW R3, y+4(FP)
1008 JMP runtime·goPanicSliceConvert(SB)
1009
1010 // Extended versions for 64-bit indexes.
1011 TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12
1012 MOVW R4, hi+0(FP)
1013 MOVW R0, lo+4(FP)
1014 MOVW R1, y+8(FP)
1015 JMP runtime·goPanicExtendIndex(SB)
1016 TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12
1017 MOVW R4, hi+0(FP)
1018 MOVW R0, lo+4(FP)
1019 MOVW R1, y+8(FP)
1020 JMP runtime·goPanicExtendIndexU(SB)
1021 TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12
1022 MOVW R4, hi+0(FP)
1023 MOVW R1, lo+4(FP)
1024 MOVW R2, y+8(FP)
1025 JMP runtime·goPanicExtendSliceAlen(SB)
1026 TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12
1027 MOVW R4, hi+0(FP)
1028 MOVW R1, lo+4(FP)
1029 MOVW R2, y+8(FP)
1030 JMP runtime·goPanicExtendSliceAlenU(SB)
1031 TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12
1032 MOVW R4, hi+0(FP)
1033 MOVW R1, lo+4(FP)
1034 MOVW R2, y+8(FP)
1035 JMP runtime·goPanicExtendSliceAcap(SB)
1036 TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12
1037 MOVW R4, hi+0(FP)
1038 MOVW R1, lo+4(FP)
1039 MOVW R2, y+8(FP)
1040 JMP runtime·goPanicExtendSliceAcapU(SB)
1041 TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12
1042 MOVW R4, hi+0(FP)
1043 MOVW R0, lo+4(FP)
1044 MOVW R1, y+8(FP)
1045 JMP runtime·goPanicExtendSliceB(SB)
1046 TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12
1047 MOVW R4, hi+0(FP)
1048 MOVW R0, lo+4(FP)
1049 MOVW R1, y+8(FP)
1050 JMP runtime·goPanicExtendSliceBU(SB)
1051 TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12
1052 MOVW R4, hi+0(FP)
1053 MOVW R2, lo+4(FP)
1054 MOVW R3, y+8(FP)
1055 JMP runtime·goPanicExtendSlice3Alen(SB)
1056 TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12
1057 MOVW R4, hi+0(FP)
1058 MOVW R2, lo+4(FP)
1059 MOVW R3, y+8(FP)
1060 JMP runtime·goPanicExtendSlice3AlenU(SB)
1061 TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12
1062 MOVW R4, hi+0(FP)
1063 MOVW R2, lo+4(FP)
1064 MOVW R3, y+8(FP)
1065 JMP runtime·goPanicExtendSlice3Acap(SB)
1066 TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12
1067 MOVW R4, hi+0(FP)
1068 MOVW R2, lo+4(FP)
1069 MOVW R3, y+8(FP)
1070 JMP runtime·goPanicExtendSlice3AcapU(SB)
1071 TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12
1072 MOVW R4, hi+0(FP)
1073 MOVW R1, lo+4(FP)
1074 MOVW R2, y+8(FP)
1075 JMP runtime·goPanicExtendSlice3B(SB)
1076 TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12
1077 MOVW R4, hi+0(FP)
1078 MOVW R1, lo+4(FP)
1079 MOVW R2, y+8(FP)
1080 JMP runtime·goPanicExtendSlice3BU(SB)
1081 TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12
1082 MOVW R4, hi+0(FP)
1083 MOVW R0, lo+4(FP)
1084 MOVW R1, y+8(FP)
1085 JMP runtime·goPanicExtendSlice3C(SB)
1086 TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12
1087 MOVW R4, hi+0(FP)
1088 MOVW R0, lo+4(FP)
1089 MOVW R1, y+8(FP)
1090 JMP runtime·goPanicExtendSlice3CU(SB)
1091
View as plain text