Black Lives Matter. Support the Equal Justice Initiative.

Text file src/runtime/asm_riscv64.s

Documentation: runtime

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "funcdata.h"
     7  #include "textflag.h"
     8  
     9  // func rt0_go()
    10  TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
    11  	// X2 = stack; A0 = argc; A1 = argv
    12  	ADD	$-24, X2
    13  	MOV	A0, 8(X2)	// argc
    14  	MOV	A1, 16(X2)	// argv
    15  
    16  	// create istack out of the given (operating system) stack.
    17  	// _cgo_init may update stackguard.
    18  	MOV	$runtime·g0(SB), g
    19  	MOV	$(-64*1024), T0
    20  	ADD	T0, X2, T1
    21  	MOV	T1, g_stackguard0(g)
    22  	MOV	T1, g_stackguard1(g)
    23  	MOV	T1, (g_stack+stack_lo)(g)
    24  	MOV	X2, (g_stack+stack_hi)(g)
    25  
    26  	// if there is a _cgo_init, call it using the gcc ABI.
    27  	MOV	_cgo_init(SB), T0
    28  	BEQ	T0, ZERO, nocgo
    29  
    30  	MOV	ZERO, A3		// arg 3: not used
    31  	MOV	ZERO, A2		// arg 2: not used
    32  	MOV	$setg_gcc<>(SB), A1	// arg 1: setg
    33  	MOV	g, A0			// arg 0: G
    34  	JALR	RA, T0
    35  
    36  nocgo:
    37  	// update stackguard after _cgo_init
    38  	MOV	(g_stack+stack_lo)(g), T0
    39  	ADD	$const__StackGuard, T0
    40  	MOV	T0, g_stackguard0(g)
    41  	MOV	T0, g_stackguard1(g)
    42  
    43  	// set the per-goroutine and per-mach "registers"
    44  	MOV	$runtime·m0(SB), T0
    45  
    46  	// save m->g0 = g0
    47  	MOV	g, m_g0(T0)
    48  	// save m0 to g0->m
    49  	MOV	T0, g_m(g)
    50  
    51  	CALL	runtime·check(SB)
    52  
    53  	// args are already prepared
    54  	CALL	runtime·args(SB)
    55  	CALL	runtime·osinit(SB)
    56  	CALL	runtime·schedinit(SB)
    57  
    58  	// create a new goroutine to start program
    59  	MOV	$runtime·mainPC(SB), T0		// entry
    60  	ADD	$-24, X2
    61  	MOV	T0, 16(X2)
    62  	MOV	ZERO, 8(X2)
    63  	MOV	ZERO, 0(X2)
    64  	CALL	runtime·newproc(SB)
    65  	ADD	$24, X2
    66  
    67  	// start this M
    68  	CALL	runtime·mstart(SB)
    69  
    70  	WORD $0 // crash if reached
    71  	RET
    72  
    73  TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
    74  	CALL	runtime·mstart0(SB)
    75  	RET // not reached
    76  
    77  // void setg_gcc(G*); set g called from gcc with g in A0
    78  TEXT setg_gcc<>(SB),NOSPLIT,$0-0
    79  	MOV	A0, g
    80  	CALL	runtime·save_g(SB)
    81  	RET
    82  
    83  // func cputicks() int64
    84  TEXT runtime·cputicks(SB),NOSPLIT,$0-8
    85  	RDTIME	A0
    86  	MOV	A0, ret+0(FP)
    87  	RET
    88  
    89  // systemstack_switch is a dummy routine that systemstack leaves at the bottom
    90  // of the G stack. We need to distinguish the routine that
    91  // lives at the bottom of the G stack from the one that lives
    92  // at the top of the system stack because the one at the top of
    93  // the system stack terminates the stack walk (see topofstack()).
    94  TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
    95  	UNDEF
    96  	JALR	RA, ZERO	// make sure this function is not leaf
    97  	RET
    98  
    99  // func systemstack(fn func())
   100  TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   101  	MOV	fn+0(FP), CTXT	// CTXT = fn
   102  	MOV	g_m(g), T0	// T0 = m
   103  
   104  	MOV	m_gsignal(T0), T1	// T1 = gsignal
   105  	BEQ	g, T1, noswitch
   106  
   107  	MOV	m_g0(T0), T1	// T1 = g0
   108  	BEQ	g, T1, noswitch
   109  
   110  	MOV	m_curg(T0), T2
   111  	BEQ	g, T2, switch
   112  
   113  	// Bad: g is not gsignal, not g0, not curg. What is it?
   114  	// Hide call from linker nosplit analysis.
   115  	MOV	$runtime·badsystemstack(SB), T1
   116  	JALR	RA, T1
   117  
   118  switch:
   119  	// save our state in g->sched. Pretend to
   120  	// be systemstack_switch if the G stack is scanned.
   121  	CALL	gosave_systemstack_switch<>(SB)
   122  
   123  	// switch to g0
   124  	MOV	T1, g
   125  	CALL	runtime·save_g(SB)
   126  	MOV	(g_sched+gobuf_sp)(g), T0
   127  	MOV	T0, X2
   128  
   129  	// call target function
   130  	MOV	0(CTXT), T1	// code pointer
   131  	JALR	RA, T1
   132  
   133  	// switch back to g
   134  	MOV	g_m(g), T0
   135  	MOV	m_curg(T0), g
   136  	CALL	runtime·save_g(SB)
   137  	MOV	(g_sched+gobuf_sp)(g), X2
   138  	MOV	ZERO, (g_sched+gobuf_sp)(g)
   139  	RET
   140  
   141  noswitch:
   142  	// already on m stack, just call directly
   143  	// Using a tail call here cleans up tracebacks since we won't stop
   144  	// at an intermediate systemstack.
   145  	MOV	0(CTXT), T1	// code pointer
   146  	ADD	$8, X2
   147  	JMP	(T1)
   148  
   149  TEXT runtime·getcallerpc(SB),NOSPLIT|NOFRAME,$0-8
   150  	MOV	0(X2), T0		// LR saved by caller
   151  	MOV	T0, ret+0(FP)
   152  	RET
   153  
   154  /*
   155   * support for morestack
   156   */
   157  
   158  // Called during function prolog when more stack is needed.
   159  // Caller has already loaded:
   160  // R1: framesize, R2: argsize, R3: LR
   161  //
   162  // The traceback routines see morestack on a g0 as being
   163  // the top of a stack (for example, morestack calling newstack
   164  // calling the scheduler calling newm calling gc), so we must
   165  // record an argument size. For that purpose, it has no arguments.
   166  
   167  // func morestack()
   168  TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
   169  	// Cannot grow scheduler stack (m->g0).
   170  	MOV	g_m(g), A0
   171  	MOV	m_g0(A0), A1
   172  	BNE	g, A1, 3(PC)
   173  	CALL	runtime·badmorestackg0(SB)
   174  	CALL	runtime·abort(SB)
   175  
   176  	// Cannot grow signal stack (m->gsignal).
   177  	MOV	m_gsignal(A0), A1
   178  	BNE	g, A1, 3(PC)
   179  	CALL	runtime·badmorestackgsignal(SB)
   180  	CALL	runtime·abort(SB)
   181  
   182  	// Called from f.
   183  	// Set g->sched to context in f.
   184  	MOV	X2, (g_sched+gobuf_sp)(g)
   185  	MOV	T0, (g_sched+gobuf_pc)(g)
   186  	MOV	RA, (g_sched+gobuf_lr)(g)
   187  	MOV	CTXT, (g_sched+gobuf_ctxt)(g)
   188  
   189  	// Called from f.
   190  	// Set m->morebuf to f's caller.
   191  	MOV	RA, (m_morebuf+gobuf_pc)(A0)	// f's caller's PC
   192  	MOV	X2, (m_morebuf+gobuf_sp)(A0)	// f's caller's SP
   193  	MOV	g, (m_morebuf+gobuf_g)(A0)
   194  
   195  	// Call newstack on m->g0's stack.
   196  	MOV	m_g0(A0), g
   197  	CALL	runtime·save_g(SB)
   198  	MOV	(g_sched+gobuf_sp)(g), X2
   199  	// Create a stack frame on g0 to call newstack.
   200  	MOV	ZERO, -8(X2)	// Zero saved LR in frame
   201  	ADD	$-8, X2
   202  	CALL	runtime·newstack(SB)
   203  
   204  	// Not reached, but make sure the return PC from the call to newstack
   205  	// is still in this function, and not the beginning of the next.
   206  	UNDEF
   207  
   208  // func morestack_noctxt()
   209  TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
   210  	MOV	ZERO, CTXT
   211  	JMP	runtime·morestack(SB)
   212  
   213  // AES hashing not implemented for riscv64
   214  TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
   215  	JMP	runtime·memhashFallback(SB)
   216  TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
   217  	JMP	runtime·strhashFallback(SB)
   218  TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
   219  	JMP	runtime·memhash32Fallback(SB)
   220  TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
   221  	JMP	runtime·memhash64Fallback(SB)
   222  
   223  // func return0()
   224  TEXT runtime·return0(SB), NOSPLIT, $0
   225  	MOV	$0, A0
   226  	RET
   227  
   228  // restore state from Gobuf; longjmp
   229  
   230  // func gogo(buf *gobuf)
   231  TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
   232  	MOV	buf+0(FP), T0
   233  	MOV	gobuf_g(T0), T1
   234  	MOV	0(T1), ZERO // make sure g != nil
   235  	JMP	gogo<>(SB)
   236  
   237  TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
   238  	MOV	T1, g
   239  	CALL	runtime·save_g(SB)
   240  
   241  	MOV	gobuf_sp(T0), X2
   242  	MOV	gobuf_lr(T0), RA
   243  	MOV	gobuf_ret(T0), A0
   244  	MOV	gobuf_ctxt(T0), CTXT
   245  	MOV	ZERO, gobuf_sp(T0)
   246  	MOV	ZERO, gobuf_ret(T0)
   247  	MOV	ZERO, gobuf_lr(T0)
   248  	MOV	ZERO, gobuf_ctxt(T0)
   249  	MOV	gobuf_pc(T0), T0
   250  	JALR	ZERO, T0
   251  
   252  // func jmpdefer(fv *funcval, argp uintptr)
   253  // called from deferreturn
   254  // 1. grab stored return address from the caller's frame
   255  // 2. sub 8 bytes to get back to JAL deferreturn
   256  // 3. JMP to fn
   257  TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
   258  	MOV	0(X2), RA
   259  	ADD	$-8, RA
   260  
   261  	MOV	fv+0(FP), CTXT
   262  	MOV	argp+8(FP), X2
   263  	ADD	$-8, X2
   264  	MOV	0(CTXT), T0
   265  	JALR	ZERO, T0
   266  
   267  // func procyield(cycles uint32)
   268  TEXT runtime·procyield(SB),NOSPLIT,$0-0
   269  	RET
   270  
   271  // Switch to m->g0's stack, call fn(g).
   272  // Fn must never return. It should gogo(&g->sched)
   273  // to keep running g.
   274  
   275  // func mcall(fn func(*g))
   276  TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
   277  	// Save caller state in g->sched
   278  	MOV	X2, (g_sched+gobuf_sp)(g)
   279  	MOV	RA, (g_sched+gobuf_pc)(g)
   280  	MOV	ZERO, (g_sched+gobuf_lr)(g)
   281  
   282  	// Switch to m->g0 & its stack, call fn.
   283  	MOV	g, T0
   284  	MOV	g_m(g), T1
   285  	MOV	m_g0(T1), g
   286  	CALL	runtime·save_g(SB)
   287  	BNE	g, T0, 2(PC)
   288  	JMP	runtime·badmcall(SB)
   289  	MOV	fn+0(FP), CTXT			// context
   290  	MOV	0(CTXT), T1			// code pointer
   291  	MOV	(g_sched+gobuf_sp)(g), X2	// sp = m->g0->sched.sp
   292  	ADD	$-16, X2
   293  	MOV	T0, 8(X2)
   294  	MOV	ZERO, 0(X2)
   295  	JALR	RA, T1
   296  	JMP	runtime·badmcall2(SB)
   297  
   298  // Save state of caller into g->sched,
   299  // but using fake PC from systemstack_switch.
   300  // Must only be called from functions with no locals ($0)
   301  // or else unwinding from systemstack_switch is incorrect.
   302  // Smashes X31.
   303  TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
   304  	MOV	$runtime·systemstack_switch(SB), X31
   305  	ADD	$8, X31	// get past prologue
   306  	MOV	X31, (g_sched+gobuf_pc)(g)
   307  	MOV	X2, (g_sched+gobuf_sp)(g)
   308  	MOV	ZERO, (g_sched+gobuf_lr)(g)
   309  	MOV	ZERO, (g_sched+gobuf_ret)(g)
   310  	// Assert ctxt is zero. See func save.
   311  	MOV	(g_sched+gobuf_ctxt)(g), X31
   312  	BEQ	ZERO, X31, 2(PC)
   313  	CALL	runtime·abort(SB)
   314  	RET
   315  
   316  // func asmcgocall(fn, arg unsafe.Pointer) int32
   317  // Call fn(arg) on the scheduler stack,
   318  // aligned appropriately for the gcc ABI.
   319  // See cgocall.go for more details.
   320  TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   321  	MOV	fn+0(FP), X5
   322  	MOV	arg+8(FP), X10
   323  
   324  	MOV	X2, X8	// save original stack pointer
   325  	MOV	g, X9
   326  
   327  	// Figure out if we need to switch to m->g0 stack.
   328  	// We get called to create new OS threads too, and those
   329  	// come in on the m->g0 stack already.
   330  	MOV	g_m(g), X6
   331  	MOV	m_g0(X6), X7
   332  	BEQ	X7, g, g0
   333  
   334  	CALL	gosave_systemstack_switch<>(SB)
   335  	MOV	X7, g
   336  	CALL	runtime·save_g(SB)
   337  	MOV	(g_sched+gobuf_sp)(g), X2
   338  
   339  	// Now on a scheduling stack (a pthread-created stack).
   340  g0:
   341  	// Save room for two of our pointers.
   342  	ADD	$-16, X2
   343  	MOV	X9, 0(X2)	// save old g on stack
   344  	MOV	(g_stack+stack_hi)(X9), X9
   345  	SUB	X8, X9, X8
   346  	MOV	X8, 8(X2)	// save depth in old g stack (can't just save SP, as stack might be copied during a callback)
   347  
   348  	JALR	RA, (X5)
   349  
   350  	// Restore g, stack pointer. X10 is return value.
   351  	MOV	0(X2), g
   352  	CALL	runtime·save_g(SB)
   353  	MOV	(g_stack+stack_hi)(g), X5
   354  	MOV	8(X2), X6
   355  	SUB	X6, X5, X6
   356  	MOV	X6, X2
   357  
   358  	MOVW	X10, ret+16(FP)
   359  	RET
   360  
   361  // func asminit()
   362  TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
   363  	RET
   364  
   365  // reflectcall: call a function with the given argument list
   366  // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   367  // we don't have variable-sized frames, so we use a small number
   368  // of constant-sized-frame functions to encode a few bits of size in the pc.
   369  // Caution: ugly multiline assembly macros in your future!
   370  
   371  #define DISPATCH(NAME,MAXSIZE)	\
   372  	MOV	$MAXSIZE, T1	\
   373  	BLTU	T1, T0, 3(PC)	\
   374  	MOV	$NAME(SB), T2;	\
   375  	JALR	ZERO, T2
   376  // Note: can't just "BR NAME(SB)" - bad inlining results.
   377  
   378  // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   379  TEXT reflect·call(SB), NOSPLIT, $0-0
   380  	JMP	·reflectcall(SB)
   381  
   382  // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   383  TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
   384  	MOVWU	frameSize+32(FP), T0
   385  	DISPATCH(runtime·call16, 16)
   386  	DISPATCH(runtime·call32, 32)
   387  	DISPATCH(runtime·call64, 64)
   388  	DISPATCH(runtime·call128, 128)
   389  	DISPATCH(runtime·call256, 256)
   390  	DISPATCH(runtime·call512, 512)
   391  	DISPATCH(runtime·call1024, 1024)
   392  	DISPATCH(runtime·call2048, 2048)
   393  	DISPATCH(runtime·call4096, 4096)
   394  	DISPATCH(runtime·call8192, 8192)
   395  	DISPATCH(runtime·call16384, 16384)
   396  	DISPATCH(runtime·call32768, 32768)
   397  	DISPATCH(runtime·call65536, 65536)
   398  	DISPATCH(runtime·call131072, 131072)
   399  	DISPATCH(runtime·call262144, 262144)
   400  	DISPATCH(runtime·call524288, 524288)
   401  	DISPATCH(runtime·call1048576, 1048576)
   402  	DISPATCH(runtime·call2097152, 2097152)
   403  	DISPATCH(runtime·call4194304, 4194304)
   404  	DISPATCH(runtime·call8388608, 8388608)
   405  	DISPATCH(runtime·call16777216, 16777216)
   406  	DISPATCH(runtime·call33554432, 33554432)
   407  	DISPATCH(runtime·call67108864, 67108864)
   408  	DISPATCH(runtime·call134217728, 134217728)
   409  	DISPATCH(runtime·call268435456, 268435456)
   410  	DISPATCH(runtime·call536870912, 536870912)
   411  	DISPATCH(runtime·call1073741824, 1073741824)
   412  	MOV	$runtime·badreflectcall(SB), T2
   413  	JALR	ZERO, T2
   414  
   415  #define CALLFN(NAME,MAXSIZE)			\
   416  TEXT NAME(SB), WRAPPER, $MAXSIZE-48;		\
   417  	NO_LOCAL_POINTERS;			\
   418  	/* copy arguments to stack */		\
   419  	MOV	stackArgs+16(FP), A1;			\
   420  	MOVWU	stackArgsSize+24(FP), A2;		\
   421  	MOV	X2, A3;				\
   422  	ADD	$8, A3;				\
   423  	ADD	A3, A2;				\
   424  	BEQ	A3, A2, 6(PC);			\
   425  	MOVBU	(A1), A4;			\
   426  	ADD	$1, A1;				\
   427  	MOVB	A4, (A3);			\
   428  	ADD	$1, A3;				\
   429  	JMP	-5(PC);				\
   430  	/* call function */			\
   431  	MOV	f+8(FP), CTXT;			\
   432  	MOV	(CTXT), A4;			\
   433  	PCDATA  $PCDATA_StackMapIndex, $0;	\
   434  	JALR	RA, A4;				\
   435  	/* copy return values back */		\
   436  	MOV	stackArgsType+0(FP), A5;		\
   437  	MOV	stackArgs+16(FP), A1;			\
   438  	MOVWU	stackArgsSize+24(FP), A2;			\
   439  	MOVWU	stackRetOffset+28(FP), A4;		\
   440  	ADD	$8, X2, A3;			\
   441  	ADD	A4, A3; 			\
   442  	ADD	A4, A1;				\
   443  	SUB	A4, A2;				\
   444  	CALL	callRet<>(SB);			\
   445  	RET
   446  
   447  // callRet copies return values back at the end of call*. This is a
   448  // separate function so it can allocate stack space for the arguments
   449  // to reflectcallmove. It does not follow the Go ABI; it expects its
   450  // arguments in registers.
   451  TEXT callRet<>(SB), NOSPLIT, $40-0
   452  	MOV	A5, 8(X2)
   453  	MOV	A1, 16(X2)
   454  	MOV	A3, 24(X2)
   455  	MOV	A2, 32(X2)
   456  	MOV	ZERO, 40(X2)
   457  	CALL	runtime·reflectcallmove(SB)
   458  	RET
   459  
   460  CALLFN(·call16, 16)
   461  CALLFN(·call32, 32)
   462  CALLFN(·call64, 64)
   463  CALLFN(·call128, 128)
   464  CALLFN(·call256, 256)
   465  CALLFN(·call512, 512)
   466  CALLFN(·call1024, 1024)
   467  CALLFN(·call2048, 2048)
   468  CALLFN(·call4096, 4096)
   469  CALLFN(·call8192, 8192)
   470  CALLFN(·call16384, 16384)
   471  CALLFN(·call32768, 32768)
   472  CALLFN(·call65536, 65536)
   473  CALLFN(·call131072, 131072)
   474  CALLFN(·call262144, 262144)
   475  CALLFN(·call524288, 524288)
   476  CALLFN(·call1048576, 1048576)
   477  CALLFN(·call2097152, 2097152)
   478  CALLFN(·call4194304, 4194304)
   479  CALLFN(·call8388608, 8388608)
   480  CALLFN(·call16777216, 16777216)
   481  CALLFN(·call33554432, 33554432)
   482  CALLFN(·call67108864, 67108864)
   483  CALLFN(·call134217728, 134217728)
   484  CALLFN(·call268435456, 268435456)
   485  CALLFN(·call536870912, 536870912)
   486  CALLFN(·call1073741824, 1073741824)
   487  
   488  // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
   489  // Must obey the gcc calling convention.
   490  TEXT _cgo_topofstack(SB),NOSPLIT,$8
   491  	// g (X27) and REG_TMP (X31) might be clobbered by load_g.
   492  	// X27 is callee-save in the gcc calling convention, so save it.
   493  	MOV	g, savedX27-8(SP)
   494  
   495  	CALL	runtime·load_g(SB)
   496  	MOV	g_m(g), X5
   497  	MOV	m_curg(X5), X5
   498  	MOV	(g_stack+stack_hi)(X5), X10 // return value in X10
   499  
   500  	MOV	savedX27-8(SP), g
   501  	RET
   502  
   503  // func goexit(neverCallThisFunction)
   504  // The top-most function running on a goroutine
   505  // returns to goexit+PCQuantum.
   506  TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
   507  	MOV	ZERO, ZERO	// NOP
   508  	JMP	runtime·goexit1(SB)	// does not return
   509  	// traceback from goexit1 must hit code range of goexit
   510  	MOV	ZERO, ZERO	// NOP
   511  
   512  // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
   513  // See cgocall.go for more details.
   514  TEXT ·cgocallback(SB),NOSPLIT,$24-24
   515  	NO_LOCAL_POINTERS
   516  
   517  	// Load m and g from thread-local storage.
   518  	MOVBU	runtime·iscgo(SB), X5
   519  	BEQ	ZERO, X5, nocgo
   520  	CALL	runtime·load_g(SB)
   521  nocgo:
   522  
   523  	// If g is nil, Go did not create the current thread.
   524  	// Call needm to obtain one for temporary use.
   525  	// In this case, we're running on the thread stack, so there's
   526  	// lots of space, but the linker doesn't know. Hide the call from
   527  	// the linker analysis by using an indirect call.
   528  	BEQ	ZERO, g, needm
   529  
   530  	MOV	g_m(g), X5
   531  	MOV	X5, savedm-8(SP)
   532  	JMP	havem
   533  
   534  needm:
   535  	MOV	g, savedm-8(SP) // g is zero, so is m.
   536  	MOV	$runtime·needm(SB), X6
   537  	JALR	RA, X6
   538  
   539  	// Set m->sched.sp = SP, so that if a panic happens
   540  	// during the function we are about to execute, it will
   541  	// have a valid SP to run on the g0 stack.
   542  	// The next few lines (after the havem label)
   543  	// will save this SP onto the stack and then write
   544  	// the same SP back to m->sched.sp. That seems redundant,
   545  	// but if an unrecovered panic happens, unwindm will
   546  	// restore the g->sched.sp from the stack location
   547  	// and then systemstack will try to use it. If we don't set it here,
   548  	// that restored SP will be uninitialized (typically 0) and
   549  	// will not be usable.
   550  	MOV	g_m(g), X5
   551  	MOV	m_g0(X5), X6
   552  	MOV	X2, (g_sched+gobuf_sp)(X6)
   553  
   554  havem:
   555  	// Now there's a valid m, and we're running on its m->g0.
   556  	// Save current m->g0->sched.sp on stack and then set it to SP.
   557  	// Save current sp in m->g0->sched.sp in preparation for
   558  	// switch back to m->curg stack.
   559  	// NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
   560  	MOV	m_g0(X5), X6
   561  	MOV	(g_sched+gobuf_sp)(X6), X7
   562  	MOV	X7, savedsp-24(SP)	// must match frame size
   563  	MOV	X2, (g_sched+gobuf_sp)(X6)
   564  
   565  	// Switch to m->curg stack and call runtime.cgocallbackg.
   566  	// Because we are taking over the execution of m->curg
   567  	// but *not* resuming what had been running, we need to
   568  	// save that information (m->curg->sched) so we can restore it.
   569  	// We can restore m->curg->sched.sp easily, because calling
   570  	// runtime.cgocallbackg leaves SP unchanged upon return.
   571  	// To save m->curg->sched.pc, we push it onto the curg stack and
   572  	// open a frame the same size as cgocallback's g0 frame.
   573  	// Once we switch to the curg stack, the pushed PC will appear
   574  	// to be the return PC of cgocallback, so that the traceback
   575  	// will seamlessly trace back into the earlier calls.
   576  	MOV	m_curg(X5), g
   577  	CALL	runtime·save_g(SB)
   578  	MOV	(g_sched+gobuf_sp)(g), X6 // prepare stack as X6
   579  	MOV	(g_sched+gobuf_pc)(g), X7
   580  	MOV	X7, -(24+8)(X6)		// "saved LR"; must match frame size
   581  	// Gather our arguments into registers.
   582  	MOV	fn+0(FP), X7
   583  	MOV	frame+8(FP), X8
   584  	MOV	ctxt+16(FP), X9
   585  	MOV	$-(24+8)(X6), X2	// switch stack; must match frame size
   586  	MOV	X7, 8(X2)
   587  	MOV	X8, 16(X2)
   588  	MOV	X9, 24(X2)
   589  	CALL	runtime·cgocallbackg(SB)
   590  
   591  	// Restore g->sched (== m->curg->sched) from saved values.
   592  	MOV	0(X2), X7
   593  	MOV	X7, (g_sched+gobuf_pc)(g)
   594  	MOV	$(24+8)(X2), X6		// must match frame size
   595  	MOV	X6, (g_sched+gobuf_sp)(g)
   596  
   597  	// Switch back to m->g0's stack and restore m->g0->sched.sp.
   598  	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
   599  	// so we do not have to restore it.)
   600  	MOV	g_m(g), X5
   601  	MOV	m_g0(X5), g
   602  	CALL	runtime·save_g(SB)
   603  	MOV	(g_sched+gobuf_sp)(g), X2
   604  	MOV	savedsp-24(SP), X6	// must match frame size
   605  	MOV	X6, (g_sched+gobuf_sp)(g)
   606  
   607  	// If the m on entry was nil, we called needm above to borrow an m
   608  	// for the duration of the call. Since the call is over, return it with dropm.
   609  	MOV	savedm-8(SP), X5
   610  	BNE	ZERO, X5, droppedm
   611  	MOV	$runtime·dropm(SB), X6
   612  	JALR	RA, X6
   613  droppedm:
   614  
   615  	// Done!
   616  	RET
   617  
   618  TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
   619  	EBREAK
   620  	RET
   621  
   622  TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
   623  	EBREAK
   624  	RET
   625  
   626  // void setg(G*); set g. for use by needm.
   627  TEXT runtime·setg(SB), NOSPLIT, $0-8
   628  	MOV	gg+0(FP), g
   629  	// This only happens if iscgo, so jump straight to save_g
   630  	CALL	runtime·save_g(SB)
   631  	RET
   632  
   633  TEXT ·checkASM(SB),NOSPLIT,$0-1
   634  	MOV	$1, T0
   635  	MOV	T0, ret+0(FP)
   636  	RET
   637  
   638  // gcWriteBarrier performs a heap pointer write and informs the GC.
   639  //
   640  // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
   641  // - T0 is the destination of the write
   642  // - T1 is the value being written at T0.
   643  // It clobbers R30 (the linker temp register - REG_TMP).
   644  // The act of CALLing gcWriteBarrier will clobber RA (LR).
   645  // It does not clobber any other general-purpose registers,
   646  // but may clobber others (e.g., floating point registers).
   647  TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$216
   648  	// Save the registers clobbered by the fast path.
   649  	MOV	A0, 25*8(X2)
   650  	MOV	A1, 26*8(X2)
   651  	MOV	g_m(g), A0
   652  	MOV	m_p(A0), A0
   653  	MOV	(p_wbBuf+wbBuf_next)(A0), A1
   654  	// Increment wbBuf.next position.
   655  	ADD	$16, A1
   656  	MOV	A1, (p_wbBuf+wbBuf_next)(A0)
   657  	MOV	(p_wbBuf+wbBuf_end)(A0), A0
   658  	MOV	A0, T6		// T6 is linker temp register (REG_TMP)
   659  	// Record the write.
   660  	MOV	T1, -16(A1)	// Record value
   661  	MOV	(T0), A0	// TODO: This turns bad writes into bad reads.
   662  	MOV	A0, -8(A1)	// Record *slot
   663  	// Is the buffer full?
   664  	BEQ	A1, T6, flush
   665  ret:
   666  	MOV	25*8(X2), A0
   667  	MOV	26*8(X2), A1
   668  	// Do the write.
   669  	MOV	T1, (T0)
   670  	RET
   671  
   672  flush:
   673  	// Save all general purpose registers since these could be
   674  	// clobbered by wbBufFlush and were not saved by the caller.
   675  	MOV	T0, 1*8(X2)	// Also first argument to wbBufFlush
   676  	MOV	T1, 2*8(X2)	// Also second argument to wbBufFlush
   677  	// X0 is zero register
   678  	// X1 is LR, saved by prologue
   679  	// X2 is SP
   680  	MOV	X3, 3*8(X2)
   681  	// X4 is TP
   682  	// X5 is first arg to wbBufFlush (T0)
   683  	// X6 is second arg to wbBufFlush (T1)
   684  	MOV	X7, 4*8(X2)
   685  	MOV	X8, 5*8(X2)
   686  	MOV	X9, 6*8(X2)
   687  	// X10 already saved (A0)
   688  	// X11 already saved (A1)
   689  	MOV	X12, 7*8(X2)
   690  	MOV	X13, 8*8(X2)
   691  	MOV	X14, 9*8(X2)
   692  	MOV	X15, 10*8(X2)
   693  	MOV	X16, 11*8(X2)
   694  	MOV	X17, 12*8(X2)
   695  	MOV	X18, 13*8(X2)
   696  	MOV	X19, 14*8(X2)
   697  	MOV	X20, 15*8(X2)
   698  	MOV	X21, 16*8(X2)
   699  	MOV	X22, 17*8(X2)
   700  	MOV	X23, 18*8(X2)
   701  	MOV	X24, 19*8(X2)
   702  	MOV	X25, 20*8(X2)
   703  	MOV	X26, 21*8(X2)
   704  	// X27 is g.
   705  	MOV	X28, 22*8(X2)
   706  	MOV	X29, 23*8(X2)
   707  	MOV	X30, 24*8(X2)
   708  	// X31 is tmp register.
   709  
   710  	// This takes arguments T0 and T1.
   711  	CALL	runtime·wbBufFlush(SB)
   712  
   713  	MOV	1*8(X2), T0
   714  	MOV	2*8(X2), T1
   715  	MOV	3*8(X2), X3
   716  	MOV	4*8(X2), X7
   717  	MOV	5*8(X2), X8
   718  	MOV	6*8(X2), X9
   719  	MOV	7*8(X2), X12
   720  	MOV	8*8(X2), X13
   721  	MOV	9*8(X2), X14
   722  	MOV	10*8(X2), X15
   723  	MOV	11*8(X2), X16
   724  	MOV	12*8(X2), X17
   725  	MOV	13*8(X2), X18
   726  	MOV	14*8(X2), X19
   727  	MOV	15*8(X2), X20
   728  	MOV	16*8(X2), X21
   729  	MOV	17*8(X2), X22
   730  	MOV	18*8(X2), X23
   731  	MOV	19*8(X2), X24
   732  	MOV	20*8(X2), X25
   733  	MOV	21*8(X2), X26
   734  	MOV	22*8(X2), X28
   735  	MOV	23*8(X2), X29
   736  	MOV	24*8(X2), X30
   737  
   738  	JMP	ret
   739  
   740  // Note: these functions use a special calling convention to save generated code space.
   741  // Arguments are passed in registers, but the space for those arguments are allocated
   742  // in the caller's stack frame. These stubs write the args into that stack space and
   743  // then tail call to the corresponding runtime handler.
   744  // The tail call makes these stubs disappear in backtraces.
   745  TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
   746  	MOV	T0, x+0(FP)
   747  	MOV	T1, y+8(FP)
   748  	JMP	runtime·goPanicIndex(SB)
   749  TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
   750  	MOV	T0, x+0(FP)
   751  	MOV	T1, y+8(FP)
   752  	JMP	runtime·goPanicIndexU(SB)
   753  TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
   754  	MOV	T1, x+0(FP)
   755  	MOV	T2, y+8(FP)
   756  	JMP	runtime·goPanicSliceAlen(SB)
   757  TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
   758  	MOV	T1, x+0(FP)
   759  	MOV	T2, y+8(FP)
   760  	JMP	runtime·goPanicSliceAlenU(SB)
   761  TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
   762  	MOV	T1, x+0(FP)
   763  	MOV	T2, y+8(FP)
   764  	JMP	runtime·goPanicSliceAcap(SB)
   765  TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
   766  	MOV	T1, x+0(FP)
   767  	MOV	T2, y+8(FP)
   768  	JMP	runtime·goPanicSliceAcapU(SB)
   769  TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
   770  	MOV	T0, x+0(FP)
   771  	MOV	T1, y+8(FP)
   772  	JMP	runtime·goPanicSliceB(SB)
   773  TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
   774  	MOV	T0, x+0(FP)
   775  	MOV	T1, y+8(FP)
   776  	JMP	runtime·goPanicSliceBU(SB)
   777  TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
   778  	MOV	T2, x+0(FP)
   779  	MOV	T3, y+8(FP)
   780  	JMP	runtime·goPanicSlice3Alen(SB)
   781  TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
   782  	MOV	T2, x+0(FP)
   783  	MOV	T3, y+8(FP)
   784  	JMP	runtime·goPanicSlice3AlenU(SB)
   785  TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
   786  	MOV	T2, x+0(FP)
   787  	MOV	T3, y+8(FP)
   788  	JMP	runtime·goPanicSlice3Acap(SB)
   789  TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
   790  	MOV	T2, x+0(FP)
   791  	MOV	T3, y+8(FP)
   792  	JMP	runtime·goPanicSlice3AcapU(SB)
   793  TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
   794  	MOV	T1, x+0(FP)
   795  	MOV	T2, y+8(FP)
   796  	JMP	runtime·goPanicSlice3B(SB)
   797  TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
   798  	MOV	T1, x+0(FP)
   799  	MOV	T2, y+8(FP)
   800  	JMP	runtime·goPanicSlice3BU(SB)
   801  TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
   802  	MOV	T0, x+0(FP)
   803  	MOV	T1, y+8(FP)
   804  	JMP	runtime·goPanicSlice3C(SB)
   805  TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
   806  	MOV	T0, x+0(FP)
   807  	MOV	T1, y+8(FP)
   808  	JMP	runtime·goPanicSlice3CU(SB)
   809  TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
   810  	MOV	T2, x+0(FP)
   811  	MOV	T3, y+8(FP)
   812  	JMP	runtime·goPanicSliceConvert(SB)
   813  
   814  DATA	runtime·mainPC+0(SB)/8,$runtime·main(SB)
   815  GLOBL	runtime·mainPC(SB),RODATA,$8
   816  

View as plain text