Text file
src/runtime/asm_ppc64x.s
Documentation: runtime
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6 // +build ppc64 ppc64le
7
8 #include "go_asm.h"
9 #include "go_tls.h"
10 #include "funcdata.h"
11 #include "textflag.h"
12 #include "asm_ppc64x.h"
13
14 #ifdef GOOS_aix
15 #define cgoCalleeStackSize 48
16 #else
17 #define cgoCalleeStackSize 32
18 #endif
19
20 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
21 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
22
23 // initialize essential registers
24 BL runtime·reginit(SB)
25
26 SUB $(FIXED_FRAME+16), R1
27 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
28 MOVW R3, FIXED_FRAME+0(R1) // argc
29 MOVD R4, FIXED_FRAME+8(R1) // argv
30
31 // create istack out of the given (operating system) stack.
32 // _cgo_init may update stackguard.
33 MOVD $runtime·g0(SB), g
34 BL runtime·save_g(SB)
35 MOVD $(-64*1024), R31
36 ADD R31, R1, R3
37 MOVD R3, g_stackguard0(g)
38 MOVD R3, g_stackguard1(g)
39 MOVD R3, (g_stack+stack_lo)(g)
40 MOVD R1, (g_stack+stack_hi)(g)
41
42 // if there is a _cgo_init, call it using the gcc ABI.
43 MOVD _cgo_init(SB), R12
44 CMP R0, R12
45 BEQ nocgo
46 #ifdef GOARCH_ppc64
47 // ppc64 use elf ABI v1. we must get the real entry address from
48 // first slot of the function descriptor before call.
49 MOVD 8(R12), R2
50 MOVD (R12), R12
51 #endif
52 MOVD R12, CTR // r12 = "global function entry point"
53 MOVD R13, R5 // arg 2: TLS base pointer
54 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
55 MOVD g, R3 // arg 0: G
56 // C functions expect 32 (48 for AIX) bytes of space on caller
57 // stack frame and a 16-byte aligned R1
58 MOVD R1, R14 // save current stack
59 SUB $cgoCalleeStackSize, R1 // reserve the callee area
60 RLDCR $0, R1, $~15, R1 // 16-byte align
61 BL (CTR) // may clobber R0, R3-R12
62 MOVD R14, R1 // restore stack
63 #ifndef GOOS_aix
64 MOVD 24(R1), R2
65 #endif
66 XOR R0, R0 // fix R0
67
68 nocgo:
69 // update stackguard after _cgo_init
70 MOVD (g_stack+stack_lo)(g), R3
71 ADD $const__StackGuard, R3
72 MOVD R3, g_stackguard0(g)
73 MOVD R3, g_stackguard1(g)
74
75 // set the per-goroutine and per-mach "registers"
76 MOVD $runtime·m0(SB), R3
77
78 // save m->g0 = g0
79 MOVD g, m_g0(R3)
80 // save m0 to g0->m
81 MOVD R3, g_m(g)
82
83 BL runtime·check(SB)
84
85 // args are already prepared
86 BL runtime·args(SB)
87 BL runtime·osinit(SB)
88 BL runtime·schedinit(SB)
89
90 // create a new goroutine to start program
91 MOVD $runtime·mainPC(SB), R3 // entry
92 MOVDU R3, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 MOVDU R0, -8(R1)
97 MOVDU R0, -8(R1)
98 BL runtime·newproc(SB)
99 ADD $(16+FIXED_FRAME), R1
100
101 // start this M
102 BL runtime·mstart(SB)
103
104 MOVD R0, 0(R0)
105 RET
106
107 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
108 GLOBL runtime·mainPC(SB),RODATA,$8
109
110 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
111 MOVD R0, 0(R0) // TODO: TD
112 RET
113
114 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
115 RET
116
117 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
118 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
119 // crosscall_ppc64 and crosscall2 need to reginit, but can't
120 // get at the 'runtime.reginit' symbol.
121 BR runtime·reginit(SB)
122
123 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
124 // set R0 to zero, it's expected by the toolchain
125 XOR R0, R0
126 RET
127
128 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
129 BL runtime·mstart0(SB)
130 RET // not reached
131
132 /*
133 * go-routine
134 */
135
136 // void gogo(Gobuf*)
137 // restore state from Gobuf; longjmp
138 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
139 MOVD buf+0(FP), R5
140 MOVD gobuf_g(R5), R6
141 MOVD 0(R6), R4 // make sure g != nil
142 BR gogo<>(SB)
143
144 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
145 MOVD R6, g
146 BL runtime·save_g(SB)
147
148 MOVD gobuf_sp(R5), R1
149 MOVD gobuf_lr(R5), R31
150 #ifndef GOOS_aix
151 MOVD 24(R1), R2 // restore R2
152 #endif
153 MOVD R31, LR
154 MOVD gobuf_ret(R5), R3
155 MOVD gobuf_ctxt(R5), R11
156 MOVD R0, gobuf_sp(R5)
157 MOVD R0, gobuf_ret(R5)
158 MOVD R0, gobuf_lr(R5)
159 MOVD R0, gobuf_ctxt(R5)
160 CMP R0, R0 // set condition codes for == test, needed by stack split
161 MOVD gobuf_pc(R5), R12
162 MOVD R12, CTR
163 BR (CTR)
164
165 // void mcall(fn func(*g))
166 // Switch to m->g0's stack, call fn(g).
167 // Fn must never return. It should gogo(&g->sched)
168 // to keep running g.
169 TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
170 // Save caller state in g->sched
171 MOVD R1, (g_sched+gobuf_sp)(g)
172 MOVD LR, R31
173 MOVD R31, (g_sched+gobuf_pc)(g)
174 MOVD R0, (g_sched+gobuf_lr)(g)
175
176 // Switch to m->g0 & its stack, call fn.
177 MOVD g, R3
178 MOVD g_m(g), R8
179 MOVD m_g0(R8), g
180 BL runtime·save_g(SB)
181 CMP g, R3
182 BNE 2(PC)
183 BR runtime·badmcall(SB)
184 MOVD fn+0(FP), R11 // context
185 MOVD 0(R11), R12 // code pointer
186 MOVD R12, CTR
187 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
188 MOVDU R3, -8(R1)
189 MOVDU R0, -8(R1)
190 MOVDU R0, -8(R1)
191 MOVDU R0, -8(R1)
192 MOVDU R0, -8(R1)
193 BL (CTR)
194 MOVD 24(R1), R2
195 BR runtime·badmcall2(SB)
196
197 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
198 // of the G stack. We need to distinguish the routine that
199 // lives at the bottom of the G stack from the one that lives
200 // at the top of the system stack because the one at the top of
201 // the system stack terminates the stack walk (see topofstack()).
202 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
203 // We have several undefs here so that 16 bytes past
204 // $runtime·systemstack_switch lies within them whether or not the
205 // instructions that derive r2 from r12 are there.
206 UNDEF
207 UNDEF
208 UNDEF
209 BL (LR) // make sure this function is not leaf
210 RET
211
212 // func systemstack(fn func())
213 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
214 MOVD fn+0(FP), R3 // R3 = fn
215 MOVD R3, R11 // context
216 MOVD g_m(g), R4 // R4 = m
217
218 MOVD m_gsignal(R4), R5 // R5 = gsignal
219 CMP g, R5
220 BEQ noswitch
221
222 MOVD m_g0(R4), R5 // R5 = g0
223 CMP g, R5
224 BEQ noswitch
225
226 MOVD m_curg(R4), R6
227 CMP g, R6
228 BEQ switch
229
230 // Bad: g is not gsignal, not g0, not curg. What is it?
231 // Hide call from linker nosplit analysis.
232 MOVD $runtime·badsystemstack(SB), R12
233 MOVD R12, CTR
234 BL (CTR)
235 BL runtime·abort(SB)
236
237 switch:
238 // save our state in g->sched. Pretend to
239 // be systemstack_switch if the G stack is scanned.
240 BL gosave_systemstack_switch<>(SB)
241
242 // switch to g0
243 MOVD R5, g
244 BL runtime·save_g(SB)
245 MOVD (g_sched+gobuf_sp)(g), R1
246
247 // call target function
248 MOVD 0(R11), R12 // code pointer
249 MOVD R12, CTR
250 BL (CTR)
251
252 // restore TOC pointer. It seems unlikely that we will use systemstack
253 // to call a function defined in another module, but the results of
254 // doing so would be so confusing that it's worth doing this.
255 MOVD g_m(g), R3
256 MOVD m_curg(R3), g
257 MOVD (g_sched+gobuf_sp)(g), R3
258 #ifndef GOOS_aix
259 MOVD 24(R3), R2
260 #endif
261 // switch back to g
262 MOVD g_m(g), R3
263 MOVD m_curg(R3), g
264 BL runtime·save_g(SB)
265 MOVD (g_sched+gobuf_sp)(g), R1
266 MOVD R0, (g_sched+gobuf_sp)(g)
267 RET
268
269 noswitch:
270 // already on m stack, just call directly
271 // On other arches we do a tail call here, but it appears to be
272 // impossible to tail call a function pointer in shared mode on
273 // ppc64 because the caller is responsible for restoring the TOC.
274 MOVD 0(R11), R12 // code pointer
275 MOVD R12, CTR
276 BL (CTR)
277 #ifndef GOOS_aix
278 MOVD 24(R1), R2
279 #endif
280 RET
281
282 /*
283 * support for morestack
284 */
285
286 // Called during function prolog when more stack is needed.
287 // Caller has already loaded:
288 // R3: framesize, R4: argsize, R5: LR
289 //
290 // The traceback routines see morestack on a g0 as being
291 // the top of a stack (for example, morestack calling newstack
292 // calling the scheduler calling newm calling gc), so we must
293 // record an argument size. For that purpose, it has no arguments.
294 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
295 // Cannot grow scheduler stack (m->g0).
296 MOVD g_m(g), R7
297 MOVD m_g0(R7), R8
298 CMP g, R8
299 BNE 3(PC)
300 BL runtime·badmorestackg0(SB)
301 BL runtime·abort(SB)
302
303 // Cannot grow signal stack (m->gsignal).
304 MOVD m_gsignal(R7), R8
305 CMP g, R8
306 BNE 3(PC)
307 BL runtime·badmorestackgsignal(SB)
308 BL runtime·abort(SB)
309
310 // Called from f.
311 // Set g->sched to context in f.
312 MOVD R1, (g_sched+gobuf_sp)(g)
313 MOVD LR, R8
314 MOVD R8, (g_sched+gobuf_pc)(g)
315 MOVD R5, (g_sched+gobuf_lr)(g)
316 MOVD R11, (g_sched+gobuf_ctxt)(g)
317
318 // Called from f.
319 // Set m->morebuf to f's caller.
320 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
321 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
322 MOVD g, (m_morebuf+gobuf_g)(R7)
323
324 // Call newstack on m->g0's stack.
325 MOVD m_g0(R7), g
326 BL runtime·save_g(SB)
327 MOVD (g_sched+gobuf_sp)(g), R1
328 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
329 BL runtime·newstack(SB)
330
331 // Not reached, but make sure the return PC from the call to newstack
332 // is still in this function, and not the beginning of the next.
333 UNDEF
334
335 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
336 MOVD R0, R11
337 BR runtime·morestack(SB)
338
339 // reflectcall: call a function with the given argument list
340 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
341 // we don't have variable-sized frames, so we use a small number
342 // of constant-sized-frame functions to encode a few bits of size in the pc.
343 // Caution: ugly multiline assembly macros in your future!
344
345 #define DISPATCH(NAME,MAXSIZE) \
346 MOVD $MAXSIZE, R31; \
347 CMP R3, R31; \
348 BGT 4(PC); \
349 MOVD $NAME(SB), R12; \
350 MOVD R12, CTR; \
351 BR (CTR)
352 // Note: can't just "BR NAME(SB)" - bad inlining results.
353
354 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
355 MOVWZ frameSize+32(FP), R3
356 DISPATCH(runtime·call16, 16)
357 DISPATCH(runtime·call32, 32)
358 DISPATCH(runtime·call64, 64)
359 DISPATCH(runtime·call128, 128)
360 DISPATCH(runtime·call256, 256)
361 DISPATCH(runtime·call512, 512)
362 DISPATCH(runtime·call1024, 1024)
363 DISPATCH(runtime·call2048, 2048)
364 DISPATCH(runtime·call4096, 4096)
365 DISPATCH(runtime·call8192, 8192)
366 DISPATCH(runtime·call16384, 16384)
367 DISPATCH(runtime·call32768, 32768)
368 DISPATCH(runtime·call65536, 65536)
369 DISPATCH(runtime·call131072, 131072)
370 DISPATCH(runtime·call262144, 262144)
371 DISPATCH(runtime·call524288, 524288)
372 DISPATCH(runtime·call1048576, 1048576)
373 DISPATCH(runtime·call2097152, 2097152)
374 DISPATCH(runtime·call4194304, 4194304)
375 DISPATCH(runtime·call8388608, 8388608)
376 DISPATCH(runtime·call16777216, 16777216)
377 DISPATCH(runtime·call33554432, 33554432)
378 DISPATCH(runtime·call67108864, 67108864)
379 DISPATCH(runtime·call134217728, 134217728)
380 DISPATCH(runtime·call268435456, 268435456)
381 DISPATCH(runtime·call536870912, 536870912)
382 DISPATCH(runtime·call1073741824, 1073741824)
383 MOVD $runtime·badreflectcall(SB), R12
384 MOVD R12, CTR
385 BR (CTR)
386
387 #define CALLFN(NAME,MAXSIZE) \
388 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
389 NO_LOCAL_POINTERS; \
390 /* copy arguments to stack */ \
391 MOVD stackArgs+16(FP), R3; \
392 MOVWZ stackArgsSize+24(FP), R4; \
393 MOVD R1, R5; \
394 CMP R4, $8; \
395 BLT tailsetup; \
396 /* copy 8 at a time if possible */ \
397 ADD $(FIXED_FRAME-8), R5; \
398 SUB $8, R3; \
399 top: \
400 MOVDU 8(R3), R7; \
401 MOVDU R7, 8(R5); \
402 SUB $8, R4; \
403 CMP R4, $8; \
404 BGE top; \
405 /* handle remaining bytes */ \
406 CMP $0, R4; \
407 BEQ callfn; \
408 ADD $7, R3; \
409 ADD $7, R5; \
410 BR tail; \
411 tailsetup: \
412 CMP $0, R4; \
413 BEQ callfn; \
414 ADD $(FIXED_FRAME-1), R5; \
415 SUB $1, R3; \
416 tail: \
417 MOVBU 1(R3), R6; \
418 MOVBU R6, 1(R5); \
419 SUB $1, R4; \
420 CMP $0, R4; \
421 BGT tail; \
422 callfn: \
423 /* call function */ \
424 MOVD f+8(FP), R11; \
425 #ifdef GOOS_aix \
426 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
427 /* So it manually triggers it */ \
428 CMP R0, R11 \
429 BNE 2(PC) \
430 MOVD R0, 0(R0) \
431 #endif \
432 MOVD (R11), R12; \
433 MOVD R12, CTR; \
434 PCDATA $PCDATA_StackMapIndex, $0; \
435 BL (CTR); \
436 #ifndef GOOS_aix \
437 MOVD 24(R1), R2; \
438 #endif \
439 /* copy return values back */ \
440 MOVD stackArgsType+0(FP), R7; \
441 MOVD stackArgs+16(FP), R3; \
442 MOVWZ stackArgsSize+24(FP), R4; \
443 MOVWZ stackRetOffset+28(FP), R6; \
444 ADD $FIXED_FRAME, R1, R5; \
445 ADD R6, R5; \
446 ADD R6, R3; \
447 SUB R6, R4; \
448 BL callRet<>(SB); \
449 RET
450
451 // callRet copies return values back at the end of call*. This is a
452 // separate function so it can allocate stack space for the arguments
453 // to reflectcallmove. It does not follow the Go ABI; it expects its
454 // arguments in registers.
455 TEXT callRet<>(SB), NOSPLIT, $40-0
456 MOVD R7, FIXED_FRAME+0(R1)
457 MOVD R3, FIXED_FRAME+8(R1)
458 MOVD R5, FIXED_FRAME+16(R1)
459 MOVD R4, FIXED_FRAME+24(R1)
460 MOVD $0, FIXED_FRAME+32(R1)
461 BL runtime·reflectcallmove(SB)
462 RET
463
464 CALLFN(·call16, 16)
465 CALLFN(·call32, 32)
466 CALLFN(·call64, 64)
467 CALLFN(·call128, 128)
468 CALLFN(·call256, 256)
469 CALLFN(·call512, 512)
470 CALLFN(·call1024, 1024)
471 CALLFN(·call2048, 2048)
472 CALLFN(·call4096, 4096)
473 CALLFN(·call8192, 8192)
474 CALLFN(·call16384, 16384)
475 CALLFN(·call32768, 32768)
476 CALLFN(·call65536, 65536)
477 CALLFN(·call131072, 131072)
478 CALLFN(·call262144, 262144)
479 CALLFN(·call524288, 524288)
480 CALLFN(·call1048576, 1048576)
481 CALLFN(·call2097152, 2097152)
482 CALLFN(·call4194304, 4194304)
483 CALLFN(·call8388608, 8388608)
484 CALLFN(·call16777216, 16777216)
485 CALLFN(·call33554432, 33554432)
486 CALLFN(·call67108864, 67108864)
487 CALLFN(·call134217728, 134217728)
488 CALLFN(·call268435456, 268435456)
489 CALLFN(·call536870912, 536870912)
490 CALLFN(·call1073741824, 1073741824)
491
492 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
493 MOVW cycles+0(FP), R7
494 // POWER does not have a pause/yield instruction equivalent.
495 // Instead, we can lower the program priority by setting the
496 // Program Priority Register prior to the wait loop and set it
497 // back to default afterwards. On Linux, the default priority is
498 // medium-low. For details, see page 837 of the ISA 3.0.
499 OR R1, R1, R1 // Set PPR priority to low
500 again:
501 SUB $1, R7
502 CMP $0, R7
503 BNE again
504 OR R6, R6, R6 // Set PPR priority back to medium-low
505 RET
506
507 // void jmpdefer(fv, sp);
508 // called from deferreturn.
509 // 1. grab stored LR for caller
510 // 2. sub 8 bytes to get back to either nop or toc reload before deferreturn
511 // 3. BR to fn
512 // When dynamically linking Go, it is not sufficient to rewind to the BL
513 // deferreturn -- we might be jumping between modules and so we need to reset
514 // the TOC pointer in r2. To do this, codegen inserts MOVD 24(R1), R2 *before*
515 // the BL deferreturn and jmpdefer rewinds to that.
516 TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
517 MOVD 0(R1), R31
518 SUB $8, R31
519 MOVD R31, LR
520
521 MOVD fv+0(FP), R11
522 MOVD argp+8(FP), R1
523 SUB $FIXED_FRAME, R1
524 #ifdef GOOS_aix
525 // AIX won't trigger a SIGSEGV if R11 = nil
526 // So it manually triggers it
527 CMP R0, R11
528 BNE 2(PC)
529 MOVD R0, 0(R0)
530 #endif
531 MOVD 0(R11), R12
532 MOVD R12, CTR
533 BR (CTR)
534
535 // Save state of caller into g->sched,
536 // but using fake PC from systemstack_switch.
537 // Must only be called from functions with no locals ($0)
538 // or else unwinding from systemstack_switch is incorrect.
539 // Smashes R31.
540 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
541 MOVD $runtime·systemstack_switch(SB), R31
542 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
543 MOVD R31, (g_sched+gobuf_pc)(g)
544 MOVD R1, (g_sched+gobuf_sp)(g)
545 MOVD R0, (g_sched+gobuf_lr)(g)
546 MOVD R0, (g_sched+gobuf_ret)(g)
547 // Assert ctxt is zero. See func save.
548 MOVD (g_sched+gobuf_ctxt)(g), R31
549 CMP R0, R31
550 BEQ 2(PC)
551 BL runtime·abort(SB)
552 RET
553
554 #ifdef GOOS_aix
555 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
556 #else
557 #define asmcgocallSaveOffset cgoCalleeStackSize
558 #endif
559
560 // func asmcgocall(fn, arg unsafe.Pointer) int32
561 // Call fn(arg) on the scheduler stack,
562 // aligned appropriately for the gcc ABI.
563 // See cgocall.go for more details.
564 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
565 MOVD fn+0(FP), R3
566 MOVD arg+8(FP), R4
567
568 MOVD R1, R7 // save original stack pointer
569 MOVD g, R5
570
571 // Figure out if we need to switch to m->g0 stack.
572 // We get called to create new OS threads too, and those
573 // come in on the m->g0 stack already.
574 // Moreover, if it's called inside the signal handler, it must not switch
575 // to g0 as it can be in use by another syscall.
576 MOVD g_m(g), R8
577 MOVD m_gsignal(R8), R6
578 CMP R6, g
579 BEQ g0
580 MOVD m_g0(R8), R6
581 CMP R6, g
582 BEQ g0
583 BL gosave_systemstack_switch<>(SB)
584 MOVD R6, g
585 BL runtime·save_g(SB)
586 MOVD (g_sched+gobuf_sp)(g), R1
587
588 // Now on a scheduling stack (a pthread-created stack).
589 g0:
590 #ifdef GOOS_aix
591 // Create a fake LR to improve backtrace.
592 MOVD $runtime·asmcgocall(SB), R6
593 MOVD R6, 16(R1)
594 // AIX also save one argument on the stack.
595 SUB $8, R1
596 #endif
597 // Save room for two of our pointers, plus the callee
598 // save area that lives on the caller stack.
599 SUB $(asmcgocallSaveOffset+16), R1
600 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
601 MOVD R5, (asmcgocallSaveOffset+8)(R1)// save old g on stack
602 MOVD (g_stack+stack_hi)(R5), R5
603 SUB R7, R5
604 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
605 #ifdef GOOS_aix
606 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
607 #else
608 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
609 #endif
610 // This is a "global call", so put the global entry point in r12
611 MOVD R3, R12
612
613 #ifdef GOARCH_ppc64
614 // ppc64 use elf ABI v1. we must get the real entry address from
615 // first slot of the function descriptor before call.
616 // Same for AIX.
617 MOVD 8(R12), R2
618 MOVD (R12), R12
619 #endif
620 MOVD R12, CTR
621 MOVD R4, R3 // arg in r3
622 BL (CTR)
623 // C code can clobber R0, so set it back to 0. F27-F31 are
624 // callee save, so we don't need to recover those.
625 XOR R0, R0
626 // Restore g, stack pointer, toc pointer.
627 // R3 is errno, so don't touch it
628 MOVD (asmcgocallSaveOffset+8)(R1), g
629 MOVD (g_stack+stack_hi)(g), R5
630 MOVD asmcgocallSaveOffset(R1), R6
631 SUB R6, R5
632 #ifndef GOOS_aix
633 MOVD 24(R5), R2
634 #endif
635 MOVD R5, R1
636 BL runtime·save_g(SB)
637
638 MOVW R3, ret+16(FP)
639 RET
640
641 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
642 // See cgocall.go for more details.
643 TEXT ·cgocallback(SB),NOSPLIT,$24-24
644 NO_LOCAL_POINTERS
645
646 // Load m and g from thread-local storage.
647 MOVBZ runtime·iscgo(SB), R3
648 CMP R3, $0
649 BEQ nocgo
650 BL runtime·load_g(SB)
651 nocgo:
652
653 // If g is nil, Go did not create the current thread.
654 // Call needm to obtain one for temporary use.
655 // In this case, we're running on the thread stack, so there's
656 // lots of space, but the linker doesn't know. Hide the call from
657 // the linker analysis by using an indirect call.
658 CMP g, $0
659 BEQ needm
660
661 MOVD g_m(g), R8
662 MOVD R8, savedm-8(SP)
663 BR havem
664
665 needm:
666 MOVD g, savedm-8(SP) // g is zero, so is m.
667 MOVD $runtime·needm(SB), R12
668 MOVD R12, CTR
669 BL (CTR)
670
671 // Set m->sched.sp = SP, so that if a panic happens
672 // during the function we are about to execute, it will
673 // have a valid SP to run on the g0 stack.
674 // The next few lines (after the havem label)
675 // will save this SP onto the stack and then write
676 // the same SP back to m->sched.sp. That seems redundant,
677 // but if an unrecovered panic happens, unwindm will
678 // restore the g->sched.sp from the stack location
679 // and then systemstack will try to use it. If we don't set it here,
680 // that restored SP will be uninitialized (typically 0) and
681 // will not be usable.
682 MOVD g_m(g), R8
683 MOVD m_g0(R8), R3
684 MOVD R1, (g_sched+gobuf_sp)(R3)
685
686 havem:
687 // Now there's a valid m, and we're running on its m->g0.
688 // Save current m->g0->sched.sp on stack and then set it to SP.
689 // Save current sp in m->g0->sched.sp in preparation for
690 // switch back to m->curg stack.
691 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
692 MOVD m_g0(R8), R3
693 MOVD (g_sched+gobuf_sp)(R3), R4
694 MOVD R4, savedsp-24(SP) // must match frame size
695 MOVD R1, (g_sched+gobuf_sp)(R3)
696
697 // Switch to m->curg stack and call runtime.cgocallbackg.
698 // Because we are taking over the execution of m->curg
699 // but *not* resuming what had been running, we need to
700 // save that information (m->curg->sched) so we can restore it.
701 // We can restore m->curg->sched.sp easily, because calling
702 // runtime.cgocallbackg leaves SP unchanged upon return.
703 // To save m->curg->sched.pc, we push it onto the curg stack and
704 // open a frame the same size as cgocallback's g0 frame.
705 // Once we switch to the curg stack, the pushed PC will appear
706 // to be the return PC of cgocallback, so that the traceback
707 // will seamlessly trace back into the earlier calls.
708 MOVD m_curg(R8), g
709 BL runtime·save_g(SB)
710 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
711 MOVD (g_sched+gobuf_pc)(g), R5
712 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
713 // Gather our arguments into registers.
714 MOVD fn+0(FP), R5
715 MOVD frame+8(FP), R6
716 MOVD ctxt+16(FP), R7
717 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
718 MOVD R5, FIXED_FRAME+0(R1)
719 MOVD R6, FIXED_FRAME+8(R1)
720 MOVD R7, FIXED_FRAME+16(R1)
721 BL runtime·cgocallbackg(SB)
722
723 // Restore g->sched (== m->curg->sched) from saved values.
724 MOVD 0(R1), R5
725 MOVD R5, (g_sched+gobuf_pc)(g)
726 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
727 MOVD R4, (g_sched+gobuf_sp)(g)
728
729 // Switch back to m->g0's stack and restore m->g0->sched.sp.
730 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
731 // so we do not have to restore it.)
732 MOVD g_m(g), R8
733 MOVD m_g0(R8), g
734 BL runtime·save_g(SB)
735 MOVD (g_sched+gobuf_sp)(g), R1
736 MOVD savedsp-24(SP), R4 // must match frame size
737 MOVD R4, (g_sched+gobuf_sp)(g)
738
739 // If the m on entry was nil, we called needm above to borrow an m
740 // for the duration of the call. Since the call is over, return it with dropm.
741 MOVD savedm-8(SP), R6
742 CMP R6, $0
743 BNE droppedm
744 MOVD $runtime·dropm(SB), R12
745 MOVD R12, CTR
746 BL (CTR)
747 droppedm:
748
749 // Done!
750 RET
751
752 // void setg(G*); set g. for use by needm.
753 TEXT runtime·setg(SB), NOSPLIT, $0-8
754 MOVD gg+0(FP), g
755 // This only happens if iscgo, so jump straight to save_g
756 BL runtime·save_g(SB)
757 RET
758
759 #ifdef GOARCH_ppc64
760 #ifdef GOOS_aix
761 DATA setg_gcc<>+0(SB)/8, $_setg_gcc<>(SB)
762 DATA setg_gcc<>+8(SB)/8, $TOC(SB)
763 DATA setg_gcc<>+16(SB)/8, $0
764 GLOBL setg_gcc<>(SB), NOPTR, $24
765 #else
766 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
767 DWORD $_setg_gcc<>(SB)
768 DWORD $0
769 DWORD $0
770 #endif
771 #endif
772
773 // void setg_gcc(G*); set g in C TLS.
774 // Must obey the gcc calling convention.
775 #ifdef GOARCH_ppc64le
776 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
777 #else
778 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
779 #endif
780 // The standard prologue clobbers R31, which is callee-save in
781 // the C ABI, so we have to use $-8-0 and save LR ourselves.
782 MOVD LR, R4
783 // Also save g and R31, since they're callee-save in C ABI
784 MOVD R31, R5
785 MOVD g, R6
786
787 MOVD R3, g
788 BL runtime·save_g(SB)
789
790 MOVD R6, g
791 MOVD R5, R31
792 MOVD R4, LR
793 RET
794
795 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
796 MOVW (R0), R0
797 UNDEF
798
799 #define TBR 268
800
801 // int64 runtime·cputicks(void)
802 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
803 MOVD SPR(TBR), R3
804 MOVD R3, ret+0(FP)
805 RET
806
807 // AES hashing not implemented for ppc64
808 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
809 JMP runtime·memhashFallback(SB)
810 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
811 JMP runtime·strhashFallback(SB)
812 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
813 JMP runtime·memhash32Fallback(SB)
814 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
815 JMP runtime·memhash64Fallback(SB)
816
817 TEXT runtime·return0(SB), NOSPLIT, $0
818 MOVW $0, R3
819 RET
820
821 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
822 // Must obey the gcc calling convention.
823 #ifdef GOOS_aix
824 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
825 // be a longcall in order to prevent trampolines from ld.
826 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
827 #else
828 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
829 #endif
830 // g (R30) and R31 are callee-save in the C ABI, so save them
831 MOVD g, R4
832 MOVD R31, R5
833 MOVD LR, R6
834
835 BL runtime·load_g(SB) // clobbers g (R30), R31
836 MOVD g_m(g), R3
837 MOVD m_curg(R3), R3
838 MOVD (g_stack+stack_hi)(R3), R3
839
840 MOVD R4, g
841 MOVD R5, R31
842 MOVD R6, LR
843 RET
844
845 // The top-most function running on a goroutine
846 // returns to goexit+PCQuantum.
847 //
848 // When dynamically linking Go, it can be returned to from a function
849 // implemented in a different module and so needs to reload the TOC pointer
850 // from the stack (although this function declares that it does not set up x-a
851 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
852 // pointer in the correct place).
853 // goexit+_PCQuantum is halfway through the usual global entry point prologue
854 // that derives r2 from r12 which is a bit silly, but not harmful.
855 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
856 MOVD 24(R1), R2
857 BL runtime·goexit1(SB) // does not return
858 // traceback from goexit1 must hit code range of goexit
859 MOVD R0, R0 // NOP
860
861 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
862 // module containing runtime) to the frame that goexit will execute in when
863 // the goroutine exits. It's implemented in assembly mainly because that's the
864 // easiest way to get access to R2.
865 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
866 MOVD sp+0(FP), R3
867 MOVD R2, 24(R3)
868 RET
869
870 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
871 ADD $-8, R1
872 MOVD R31, 0(R1)
873 MOVD runtime·lastmoduledatap(SB), R4
874 MOVD R3, moduledata_next(R4)
875 MOVD R3, runtime·lastmoduledatap(SB)
876 MOVD 0(R1), R31
877 ADD $8, R1
878 RET
879
880 TEXT ·checkASM(SB),NOSPLIT,$0-1
881 MOVW $1, R3
882 MOVB R3, ret+0(FP)
883 RET
884
885 // gcWriteBarrier performs a heap pointer write and informs the GC.
886 //
887 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
888 // - R20 is the destination of the write
889 // - R21 is the value being written at R20.
890 // It clobbers condition codes.
891 // It does not clobber R0 through R17 (except special registers),
892 // but may clobber any other register, *including* R31.
893 TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112
894 // The standard prologue clobbers R31.
895 // We use R18 and R19 as scratch registers.
896 MOVD g_m(g), R18
897 MOVD m_p(R18), R18
898 MOVD (p_wbBuf+wbBuf_next)(R18), R19
899 // Increment wbBuf.next position.
900 ADD $16, R19
901 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
902 MOVD (p_wbBuf+wbBuf_end)(R18), R18
903 CMP R18, R19
904 // Record the write.
905 MOVD R21, -16(R19) // Record value
906 MOVD (R20), R18 // TODO: This turns bad writes into bad reads.
907 MOVD R18, -8(R19) // Record *slot
908 // Is the buffer full? (flags set in CMP above)
909 BEQ flush
910 ret:
911 // Do the write.
912 MOVD R21, (R20)
913 RET
914
915 flush:
916 // Save registers R0 through R15 since these were not saved by the caller.
917 // We don't save all registers on ppc64 because it takes too much space.
918 MOVD R20, (FIXED_FRAME+0)(R1) // Also first argument to wbBufFlush
919 MOVD R21, (FIXED_FRAME+8)(R1) // Also second argument to wbBufFlush
920 // R0 is always 0, so no need to spill.
921 // R1 is SP.
922 // R2 is SB.
923 MOVD R3, (FIXED_FRAME+16)(R1)
924 MOVD R4, (FIXED_FRAME+24)(R1)
925 MOVD R5, (FIXED_FRAME+32)(R1)
926 MOVD R6, (FIXED_FRAME+40)(R1)
927 MOVD R7, (FIXED_FRAME+48)(R1)
928 MOVD R8, (FIXED_FRAME+56)(R1)
929 MOVD R9, (FIXED_FRAME+64)(R1)
930 MOVD R10, (FIXED_FRAME+72)(R1)
931 // R11, R12 may be clobbered by external-linker-inserted trampoline
932 // R13 is REGTLS
933 MOVD R14, (FIXED_FRAME+80)(R1)
934 MOVD R15, (FIXED_FRAME+88)(R1)
935 MOVD R16, (FIXED_FRAME+96)(R1)
936 MOVD R17, (FIXED_FRAME+104)(R1)
937
938 // This takes arguments R20 and R21.
939 CALL runtime·wbBufFlush(SB)
940
941 MOVD (FIXED_FRAME+0)(R1), R20
942 MOVD (FIXED_FRAME+8)(R1), R21
943 MOVD (FIXED_FRAME+16)(R1), R3
944 MOVD (FIXED_FRAME+24)(R1), R4
945 MOVD (FIXED_FRAME+32)(R1), R5
946 MOVD (FIXED_FRAME+40)(R1), R6
947 MOVD (FIXED_FRAME+48)(R1), R7
948 MOVD (FIXED_FRAME+56)(R1), R8
949 MOVD (FIXED_FRAME+64)(R1), R9
950 MOVD (FIXED_FRAME+72)(R1), R10
951 MOVD (FIXED_FRAME+80)(R1), R14
952 MOVD (FIXED_FRAME+88)(R1), R15
953 MOVD (FIXED_FRAME+96)(R1), R16
954 MOVD (FIXED_FRAME+104)(R1), R17
955 JMP ret
956
957 // Note: these functions use a special calling convention to save generated code space.
958 // Arguments are passed in registers, but the space for those arguments are allocated
959 // in the caller's stack frame. These stubs write the args into that stack space and
960 // then tail call to the corresponding runtime handler.
961 // The tail call makes these stubs disappear in backtraces.
962 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
963 MOVD R3, x+0(FP)
964 MOVD R4, y+8(FP)
965 JMP runtime·goPanicIndex(SB)
966 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
967 MOVD R3, x+0(FP)
968 MOVD R4, y+8(FP)
969 JMP runtime·goPanicIndexU(SB)
970 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
971 MOVD R4, x+0(FP)
972 MOVD R5, y+8(FP)
973 JMP runtime·goPanicSliceAlen(SB)
974 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
975 MOVD R4, x+0(FP)
976 MOVD R5, y+8(FP)
977 JMP runtime·goPanicSliceAlenU(SB)
978 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
979 MOVD R4, x+0(FP)
980 MOVD R5, y+8(FP)
981 JMP runtime·goPanicSliceAcap(SB)
982 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
983 MOVD R4, x+0(FP)
984 MOVD R5, y+8(FP)
985 JMP runtime·goPanicSliceAcapU(SB)
986 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
987 MOVD R3, x+0(FP)
988 MOVD R4, y+8(FP)
989 JMP runtime·goPanicSliceB(SB)
990 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
991 MOVD R3, x+0(FP)
992 MOVD R4, y+8(FP)
993 JMP runtime·goPanicSliceBU(SB)
994 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
995 MOVD R5, x+0(FP)
996 MOVD R6, y+8(FP)
997 JMP runtime·goPanicSlice3Alen(SB)
998 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
999 MOVD R5, x+0(FP)
1000 MOVD R6, y+8(FP)
1001 JMP runtime·goPanicSlice3AlenU(SB)
1002 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
1003 MOVD R5, x+0(FP)
1004 MOVD R6, y+8(FP)
1005 JMP runtime·goPanicSlice3Acap(SB)
1006 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
1007 MOVD R5, x+0(FP)
1008 MOVD R6, y+8(FP)
1009 JMP runtime·goPanicSlice3AcapU(SB)
1010 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
1011 MOVD R4, x+0(FP)
1012 MOVD R5, y+8(FP)
1013 JMP runtime·goPanicSlice3B(SB)
1014 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
1015 MOVD R4, x+0(FP)
1016 MOVD R5, y+8(FP)
1017 JMP runtime·goPanicSlice3BU(SB)
1018 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
1019 MOVD R3, x+0(FP)
1020 MOVD R4, y+8(FP)
1021 JMP runtime·goPanicSlice3C(SB)
1022 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
1023 MOVD R3, x+0(FP)
1024 MOVD R4, y+8(FP)
1025 JMP runtime·goPanicSlice3CU(SB)
1026 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
1027 MOVD R5, x+0(FP)
1028 MOVD R6, y+8(FP)
1029 JMP runtime·goPanicSliceConvert(SB)
1030
View as plain text