Source file src/cmd/compile/internal/arm64/ssa.go

Documentation: cmd/compile/internal/arm64

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package arm64
     6  
     7  import (
     8  	"math"
     9  
    10  	"cmd/compile/internal/base"
    11  	"cmd/compile/internal/ir"
    12  	"cmd/compile/internal/logopt"
    13  	"cmd/compile/internal/ssa"
    14  	"cmd/compile/internal/ssagen"
    15  	"cmd/compile/internal/types"
    16  	"cmd/internal/obj"
    17  	"cmd/internal/obj/arm64"
    18  )
    19  
    20  // loadByType returns the load instruction of the given type.
    21  func loadByType(t *types.Type) obj.As {
    22  	if t.IsFloat() {
    23  		switch t.Size() {
    24  		case 4:
    25  			return arm64.AFMOVS
    26  		case 8:
    27  			return arm64.AFMOVD
    28  		}
    29  	} else {
    30  		switch t.Size() {
    31  		case 1:
    32  			if t.IsSigned() {
    33  				return arm64.AMOVB
    34  			} else {
    35  				return arm64.AMOVBU
    36  			}
    37  		case 2:
    38  			if t.IsSigned() {
    39  				return arm64.AMOVH
    40  			} else {
    41  				return arm64.AMOVHU
    42  			}
    43  		case 4:
    44  			if t.IsSigned() {
    45  				return arm64.AMOVW
    46  			} else {
    47  				return arm64.AMOVWU
    48  			}
    49  		case 8:
    50  			return arm64.AMOVD
    51  		}
    52  	}
    53  	panic("bad load type")
    54  }
    55  
    56  // storeByType returns the store instruction of the given type.
    57  func storeByType(t *types.Type) obj.As {
    58  	if t.IsFloat() {
    59  		switch t.Size() {
    60  		case 4:
    61  			return arm64.AFMOVS
    62  		case 8:
    63  			return arm64.AFMOVD
    64  		}
    65  	} else {
    66  		switch t.Size() {
    67  		case 1:
    68  			return arm64.AMOVB
    69  		case 2:
    70  			return arm64.AMOVH
    71  		case 4:
    72  			return arm64.AMOVW
    73  		case 8:
    74  			return arm64.AMOVD
    75  		}
    76  	}
    77  	panic("bad store type")
    78  }
    79  
    80  // makeshift encodes a register shifted by a constant, used as an Offset in Prog
    81  func makeshift(reg int16, typ int64, s int64) int64 {
    82  	return int64(reg&31)<<16 | typ | (s&63)<<10
    83  }
    84  
    85  // genshift generates a Prog for r = r0 op (r1 shifted by n)
    86  func genshift(s *ssagen.State, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog {
    87  	p := s.Prog(as)
    88  	p.From.Type = obj.TYPE_SHIFT
    89  	p.From.Offset = makeshift(r1, typ, n)
    90  	p.Reg = r0
    91  	if r != 0 {
    92  		p.To.Type = obj.TYPE_REG
    93  		p.To.Reg = r
    94  	}
    95  	return p
    96  }
    97  
    98  // generate the memory operand for the indexed load/store instructions
    99  func genIndexedOperand(v *ssa.Value) obj.Addr {
   100  	// Reg: base register, Index: (shifted) index register
   101  	mop := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   102  	switch v.Op {
   103  	case ssa.OpARM64MOVDloadidx8, ssa.OpARM64MOVDstoreidx8, ssa.OpARM64MOVDstorezeroidx8,
   104  		ssa.OpARM64FMOVDloadidx8, ssa.OpARM64FMOVDstoreidx8:
   105  		mop.Index = arm64.REG_LSL | 3<<5 | v.Args[1].Reg()&31
   106  	case ssa.OpARM64MOVWloadidx4, ssa.OpARM64MOVWUloadidx4, ssa.OpARM64MOVWstoreidx4, ssa.OpARM64MOVWstorezeroidx4,
   107  		ssa.OpARM64FMOVSloadidx4, ssa.OpARM64FMOVSstoreidx4:
   108  		mop.Index = arm64.REG_LSL | 2<<5 | v.Args[1].Reg()&31
   109  	case ssa.OpARM64MOVHloadidx2, ssa.OpARM64MOVHUloadidx2, ssa.OpARM64MOVHstoreidx2, ssa.OpARM64MOVHstorezeroidx2:
   110  		mop.Index = arm64.REG_LSL | 1<<5 | v.Args[1].Reg()&31
   111  	default: // not shifted
   112  		mop.Index = v.Args[1].Reg()
   113  	}
   114  	return mop
   115  }
   116  
   117  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   118  	switch v.Op {
   119  	case ssa.OpCopy, ssa.OpARM64MOVDreg:
   120  		if v.Type.IsMemory() {
   121  			return
   122  		}
   123  		x := v.Args[0].Reg()
   124  		y := v.Reg()
   125  		if x == y {
   126  			return
   127  		}
   128  		as := arm64.AMOVD
   129  		if v.Type.IsFloat() {
   130  			switch v.Type.Size() {
   131  			case 4:
   132  				as = arm64.AFMOVS
   133  			case 8:
   134  				as = arm64.AFMOVD
   135  			default:
   136  				panic("bad float size")
   137  			}
   138  		}
   139  		p := s.Prog(as)
   140  		p.From.Type = obj.TYPE_REG
   141  		p.From.Reg = x
   142  		p.To.Type = obj.TYPE_REG
   143  		p.To.Reg = y
   144  	case ssa.OpARM64MOVDnop:
   145  		// nothing to do
   146  	case ssa.OpLoadReg:
   147  		if v.Type.IsFlags() {
   148  			v.Fatalf("load flags not implemented: %v", v.LongString())
   149  			return
   150  		}
   151  		p := s.Prog(loadByType(v.Type))
   152  		ssagen.AddrAuto(&p.From, v.Args[0])
   153  		p.To.Type = obj.TYPE_REG
   154  		p.To.Reg = v.Reg()
   155  	case ssa.OpStoreReg:
   156  		if v.Type.IsFlags() {
   157  			v.Fatalf("store flags not implemented: %v", v.LongString())
   158  			return
   159  		}
   160  		p := s.Prog(storeByType(v.Type))
   161  		p.From.Type = obj.TYPE_REG
   162  		p.From.Reg = v.Args[0].Reg()
   163  		ssagen.AddrAuto(&p.To, v)
   164  	case ssa.OpARM64ADD,
   165  		ssa.OpARM64SUB,
   166  		ssa.OpARM64AND,
   167  		ssa.OpARM64OR,
   168  		ssa.OpARM64XOR,
   169  		ssa.OpARM64BIC,
   170  		ssa.OpARM64EON,
   171  		ssa.OpARM64ORN,
   172  		ssa.OpARM64MUL,
   173  		ssa.OpARM64MULW,
   174  		ssa.OpARM64MNEG,
   175  		ssa.OpARM64MNEGW,
   176  		ssa.OpARM64MULH,
   177  		ssa.OpARM64UMULH,
   178  		ssa.OpARM64MULL,
   179  		ssa.OpARM64UMULL,
   180  		ssa.OpARM64DIV,
   181  		ssa.OpARM64UDIV,
   182  		ssa.OpARM64DIVW,
   183  		ssa.OpARM64UDIVW,
   184  		ssa.OpARM64MOD,
   185  		ssa.OpARM64UMOD,
   186  		ssa.OpARM64MODW,
   187  		ssa.OpARM64UMODW,
   188  		ssa.OpARM64SLL,
   189  		ssa.OpARM64SRL,
   190  		ssa.OpARM64SRA,
   191  		ssa.OpARM64FADDS,
   192  		ssa.OpARM64FADDD,
   193  		ssa.OpARM64FSUBS,
   194  		ssa.OpARM64FSUBD,
   195  		ssa.OpARM64FMULS,
   196  		ssa.OpARM64FMULD,
   197  		ssa.OpARM64FNMULS,
   198  		ssa.OpARM64FNMULD,
   199  		ssa.OpARM64FDIVS,
   200  		ssa.OpARM64FDIVD,
   201  		ssa.OpARM64ROR,
   202  		ssa.OpARM64RORW:
   203  		r := v.Reg()
   204  		r1 := v.Args[0].Reg()
   205  		r2 := v.Args[1].Reg()
   206  		p := s.Prog(v.Op.Asm())
   207  		p.From.Type = obj.TYPE_REG
   208  		p.From.Reg = r2
   209  		p.Reg = r1
   210  		p.To.Type = obj.TYPE_REG
   211  		p.To.Reg = r
   212  	case ssa.OpARM64FMADDS,
   213  		ssa.OpARM64FMADDD,
   214  		ssa.OpARM64FNMADDS,
   215  		ssa.OpARM64FNMADDD,
   216  		ssa.OpARM64FMSUBS,
   217  		ssa.OpARM64FMSUBD,
   218  		ssa.OpARM64FNMSUBS,
   219  		ssa.OpARM64FNMSUBD,
   220  		ssa.OpARM64MADD,
   221  		ssa.OpARM64MADDW,
   222  		ssa.OpARM64MSUB,
   223  		ssa.OpARM64MSUBW:
   224  		rt := v.Reg()
   225  		ra := v.Args[0].Reg()
   226  		rm := v.Args[1].Reg()
   227  		rn := v.Args[2].Reg()
   228  		p := s.Prog(v.Op.Asm())
   229  		p.Reg = ra
   230  		p.From.Type = obj.TYPE_REG
   231  		p.From.Reg = rm
   232  		p.SetFrom3Reg(rn)
   233  		p.To.Type = obj.TYPE_REG
   234  		p.To.Reg = rt
   235  	case ssa.OpARM64ADDconst,
   236  		ssa.OpARM64SUBconst,
   237  		ssa.OpARM64ANDconst,
   238  		ssa.OpARM64ORconst,
   239  		ssa.OpARM64XORconst,
   240  		ssa.OpARM64SLLconst,
   241  		ssa.OpARM64SRLconst,
   242  		ssa.OpARM64SRAconst,
   243  		ssa.OpARM64RORconst,
   244  		ssa.OpARM64RORWconst:
   245  		p := s.Prog(v.Op.Asm())
   246  		p.From.Type = obj.TYPE_CONST
   247  		p.From.Offset = v.AuxInt
   248  		p.Reg = v.Args[0].Reg()
   249  		p.To.Type = obj.TYPE_REG
   250  		p.To.Reg = v.Reg()
   251  	case ssa.OpARM64ADDSconstflags:
   252  		p := s.Prog(v.Op.Asm())
   253  		p.From.Type = obj.TYPE_CONST
   254  		p.From.Offset = v.AuxInt
   255  		p.Reg = v.Args[0].Reg()
   256  		p.To.Type = obj.TYPE_REG
   257  		p.To.Reg = v.Reg0()
   258  	case ssa.OpARM64ADCzerocarry:
   259  		p := s.Prog(v.Op.Asm())
   260  		p.From.Type = obj.TYPE_REG
   261  		p.From.Reg = arm64.REGZERO
   262  		p.Reg = arm64.REGZERO
   263  		p.To.Type = obj.TYPE_REG
   264  		p.To.Reg = v.Reg()
   265  	case ssa.OpARM64ADCSflags,
   266  		ssa.OpARM64ADDSflags,
   267  		ssa.OpARM64SBCSflags,
   268  		ssa.OpARM64SUBSflags:
   269  		r := v.Reg0()
   270  		r1 := v.Args[0].Reg()
   271  		r2 := v.Args[1].Reg()
   272  		p := s.Prog(v.Op.Asm())
   273  		p.From.Type = obj.TYPE_REG
   274  		p.From.Reg = r2
   275  		p.Reg = r1
   276  		p.To.Type = obj.TYPE_REG
   277  		p.To.Reg = r
   278  	case ssa.OpARM64NEGSflags:
   279  		p := s.Prog(v.Op.Asm())
   280  		p.From.Type = obj.TYPE_REG
   281  		p.From.Reg = v.Args[0].Reg()
   282  		p.To.Type = obj.TYPE_REG
   283  		p.To.Reg = v.Reg0()
   284  	case ssa.OpARM64NGCzerocarry:
   285  		p := s.Prog(v.Op.Asm())
   286  		p.From.Type = obj.TYPE_REG
   287  		p.From.Reg = arm64.REGZERO
   288  		p.To.Type = obj.TYPE_REG
   289  		p.To.Reg = v.Reg()
   290  	case ssa.OpARM64EXTRconst,
   291  		ssa.OpARM64EXTRWconst:
   292  		p := s.Prog(v.Op.Asm())
   293  		p.From.Type = obj.TYPE_CONST
   294  		p.From.Offset = v.AuxInt
   295  		p.SetFrom3Reg(v.Args[0].Reg())
   296  		p.Reg = v.Args[1].Reg()
   297  		p.To.Type = obj.TYPE_REG
   298  		p.To.Reg = v.Reg()
   299  	case ssa.OpARM64MVNshiftLL, ssa.OpARM64NEGshiftLL:
   300  		genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
   301  	case ssa.OpARM64MVNshiftRL, ssa.OpARM64NEGshiftRL:
   302  		genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
   303  	case ssa.OpARM64MVNshiftRA, ssa.OpARM64NEGshiftRA:
   304  		genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
   305  	case ssa.OpARM64ADDshiftLL,
   306  		ssa.OpARM64SUBshiftLL,
   307  		ssa.OpARM64ANDshiftLL,
   308  		ssa.OpARM64ORshiftLL,
   309  		ssa.OpARM64XORshiftLL,
   310  		ssa.OpARM64EONshiftLL,
   311  		ssa.OpARM64ORNshiftLL,
   312  		ssa.OpARM64BICshiftLL:
   313  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
   314  	case ssa.OpARM64ADDshiftRL,
   315  		ssa.OpARM64SUBshiftRL,
   316  		ssa.OpARM64ANDshiftRL,
   317  		ssa.OpARM64ORshiftRL,
   318  		ssa.OpARM64XORshiftRL,
   319  		ssa.OpARM64EONshiftRL,
   320  		ssa.OpARM64ORNshiftRL,
   321  		ssa.OpARM64BICshiftRL:
   322  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
   323  	case ssa.OpARM64ADDshiftRA,
   324  		ssa.OpARM64SUBshiftRA,
   325  		ssa.OpARM64ANDshiftRA,
   326  		ssa.OpARM64ORshiftRA,
   327  		ssa.OpARM64XORshiftRA,
   328  		ssa.OpARM64EONshiftRA,
   329  		ssa.OpARM64ORNshiftRA,
   330  		ssa.OpARM64BICshiftRA:
   331  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
   332  	case ssa.OpARM64MOVDconst:
   333  		p := s.Prog(v.Op.Asm())
   334  		p.From.Type = obj.TYPE_CONST
   335  		p.From.Offset = v.AuxInt
   336  		p.To.Type = obj.TYPE_REG
   337  		p.To.Reg = v.Reg()
   338  	case ssa.OpARM64FMOVSconst,
   339  		ssa.OpARM64FMOVDconst:
   340  		p := s.Prog(v.Op.Asm())
   341  		p.From.Type = obj.TYPE_FCONST
   342  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   343  		p.To.Type = obj.TYPE_REG
   344  		p.To.Reg = v.Reg()
   345  	case ssa.OpARM64FCMPS0,
   346  		ssa.OpARM64FCMPD0:
   347  		p := s.Prog(v.Op.Asm())
   348  		p.From.Type = obj.TYPE_FCONST
   349  		p.From.Val = math.Float64frombits(0)
   350  		p.Reg = v.Args[0].Reg()
   351  	case ssa.OpARM64CMP,
   352  		ssa.OpARM64CMPW,
   353  		ssa.OpARM64CMN,
   354  		ssa.OpARM64CMNW,
   355  		ssa.OpARM64TST,
   356  		ssa.OpARM64TSTW,
   357  		ssa.OpARM64FCMPS,
   358  		ssa.OpARM64FCMPD:
   359  		p := s.Prog(v.Op.Asm())
   360  		p.From.Type = obj.TYPE_REG
   361  		p.From.Reg = v.Args[1].Reg()
   362  		p.Reg = v.Args[0].Reg()
   363  	case ssa.OpARM64CMPconst,
   364  		ssa.OpARM64CMPWconst,
   365  		ssa.OpARM64CMNconst,
   366  		ssa.OpARM64CMNWconst,
   367  		ssa.OpARM64TSTconst,
   368  		ssa.OpARM64TSTWconst:
   369  		p := s.Prog(v.Op.Asm())
   370  		p.From.Type = obj.TYPE_CONST
   371  		p.From.Offset = v.AuxInt
   372  		p.Reg = v.Args[0].Reg()
   373  	case ssa.OpARM64CMPshiftLL, ssa.OpARM64CMNshiftLL, ssa.OpARM64TSTshiftLL:
   374  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt)
   375  	case ssa.OpARM64CMPshiftRL, ssa.OpARM64CMNshiftRL, ssa.OpARM64TSTshiftRL:
   376  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt)
   377  	case ssa.OpARM64CMPshiftRA, ssa.OpARM64CMNshiftRA, ssa.OpARM64TSTshiftRA:
   378  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt)
   379  	case ssa.OpARM64MOVDaddr:
   380  		p := s.Prog(arm64.AMOVD)
   381  		p.From.Type = obj.TYPE_ADDR
   382  		p.From.Reg = v.Args[0].Reg()
   383  		p.To.Type = obj.TYPE_REG
   384  		p.To.Reg = v.Reg()
   385  
   386  		var wantreg string
   387  		// MOVD $sym+off(base), R
   388  		// the assembler expands it as the following:
   389  		// - base is SP: add constant offset to SP (R13)
   390  		//               when constant is large, tmp register (R11) may be used
   391  		// - base is SB: load external address from constant pool (use relocation)
   392  		switch v.Aux.(type) {
   393  		default:
   394  			v.Fatalf("aux is of unknown type %T", v.Aux)
   395  		case *obj.LSym:
   396  			wantreg = "SB"
   397  			ssagen.AddAux(&p.From, v)
   398  		case *ir.Name:
   399  			wantreg = "SP"
   400  			ssagen.AddAux(&p.From, v)
   401  		case nil:
   402  			// No sym, just MOVD $off(SP), R
   403  			wantreg = "SP"
   404  			p.From.Offset = v.AuxInt
   405  		}
   406  		if reg := v.Args[0].RegName(); reg != wantreg {
   407  			v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
   408  		}
   409  	case ssa.OpARM64MOVBload,
   410  		ssa.OpARM64MOVBUload,
   411  		ssa.OpARM64MOVHload,
   412  		ssa.OpARM64MOVHUload,
   413  		ssa.OpARM64MOVWload,
   414  		ssa.OpARM64MOVWUload,
   415  		ssa.OpARM64MOVDload,
   416  		ssa.OpARM64FMOVSload,
   417  		ssa.OpARM64FMOVDload:
   418  		p := s.Prog(v.Op.Asm())
   419  		p.From.Type = obj.TYPE_MEM
   420  		p.From.Reg = v.Args[0].Reg()
   421  		ssagen.AddAux(&p.From, v)
   422  		p.To.Type = obj.TYPE_REG
   423  		p.To.Reg = v.Reg()
   424  	case ssa.OpARM64MOVBloadidx,
   425  		ssa.OpARM64MOVBUloadidx,
   426  		ssa.OpARM64MOVHloadidx,
   427  		ssa.OpARM64MOVHUloadidx,
   428  		ssa.OpARM64MOVWloadidx,
   429  		ssa.OpARM64MOVWUloadidx,
   430  		ssa.OpARM64MOVDloadidx,
   431  		ssa.OpARM64FMOVSloadidx,
   432  		ssa.OpARM64FMOVDloadidx,
   433  		ssa.OpARM64MOVHloadidx2,
   434  		ssa.OpARM64MOVHUloadidx2,
   435  		ssa.OpARM64MOVWloadidx4,
   436  		ssa.OpARM64MOVWUloadidx4,
   437  		ssa.OpARM64MOVDloadidx8,
   438  		ssa.OpARM64FMOVDloadidx8,
   439  		ssa.OpARM64FMOVSloadidx4:
   440  		p := s.Prog(v.Op.Asm())
   441  		p.From = genIndexedOperand(v)
   442  		p.To.Type = obj.TYPE_REG
   443  		p.To.Reg = v.Reg()
   444  	case ssa.OpARM64LDAR,
   445  		ssa.OpARM64LDARB,
   446  		ssa.OpARM64LDARW:
   447  		p := s.Prog(v.Op.Asm())
   448  		p.From.Type = obj.TYPE_MEM
   449  		p.From.Reg = v.Args[0].Reg()
   450  		ssagen.AddAux(&p.From, v)
   451  		p.To.Type = obj.TYPE_REG
   452  		p.To.Reg = v.Reg0()
   453  	case ssa.OpARM64MOVBstore,
   454  		ssa.OpARM64MOVHstore,
   455  		ssa.OpARM64MOVWstore,
   456  		ssa.OpARM64MOVDstore,
   457  		ssa.OpARM64FMOVSstore,
   458  		ssa.OpARM64FMOVDstore,
   459  		ssa.OpARM64STLRB,
   460  		ssa.OpARM64STLR,
   461  		ssa.OpARM64STLRW:
   462  		p := s.Prog(v.Op.Asm())
   463  		p.From.Type = obj.TYPE_REG
   464  		p.From.Reg = v.Args[1].Reg()
   465  		p.To.Type = obj.TYPE_MEM
   466  		p.To.Reg = v.Args[0].Reg()
   467  		ssagen.AddAux(&p.To, v)
   468  	case ssa.OpARM64MOVBstoreidx,
   469  		ssa.OpARM64MOVHstoreidx,
   470  		ssa.OpARM64MOVWstoreidx,
   471  		ssa.OpARM64MOVDstoreidx,
   472  		ssa.OpARM64FMOVSstoreidx,
   473  		ssa.OpARM64FMOVDstoreidx,
   474  		ssa.OpARM64MOVHstoreidx2,
   475  		ssa.OpARM64MOVWstoreidx4,
   476  		ssa.OpARM64FMOVSstoreidx4,
   477  		ssa.OpARM64MOVDstoreidx8,
   478  		ssa.OpARM64FMOVDstoreidx8:
   479  		p := s.Prog(v.Op.Asm())
   480  		p.To = genIndexedOperand(v)
   481  		p.From.Type = obj.TYPE_REG
   482  		p.From.Reg = v.Args[2].Reg()
   483  	case ssa.OpARM64STP:
   484  		p := s.Prog(v.Op.Asm())
   485  		p.From.Type = obj.TYPE_REGREG
   486  		p.From.Reg = v.Args[1].Reg()
   487  		p.From.Offset = int64(v.Args[2].Reg())
   488  		p.To.Type = obj.TYPE_MEM
   489  		p.To.Reg = v.Args[0].Reg()
   490  		ssagen.AddAux(&p.To, v)
   491  	case ssa.OpARM64MOVBstorezero,
   492  		ssa.OpARM64MOVHstorezero,
   493  		ssa.OpARM64MOVWstorezero,
   494  		ssa.OpARM64MOVDstorezero:
   495  		p := s.Prog(v.Op.Asm())
   496  		p.From.Type = obj.TYPE_REG
   497  		p.From.Reg = arm64.REGZERO
   498  		p.To.Type = obj.TYPE_MEM
   499  		p.To.Reg = v.Args[0].Reg()
   500  		ssagen.AddAux(&p.To, v)
   501  	case ssa.OpARM64MOVBstorezeroidx,
   502  		ssa.OpARM64MOVHstorezeroidx,
   503  		ssa.OpARM64MOVWstorezeroidx,
   504  		ssa.OpARM64MOVDstorezeroidx,
   505  		ssa.OpARM64MOVHstorezeroidx2,
   506  		ssa.OpARM64MOVWstorezeroidx4,
   507  		ssa.OpARM64MOVDstorezeroidx8:
   508  		p := s.Prog(v.Op.Asm())
   509  		p.To = genIndexedOperand(v)
   510  		p.From.Type = obj.TYPE_REG
   511  		p.From.Reg = arm64.REGZERO
   512  	case ssa.OpARM64MOVQstorezero:
   513  		p := s.Prog(v.Op.Asm())
   514  		p.From.Type = obj.TYPE_REGREG
   515  		p.From.Reg = arm64.REGZERO
   516  		p.From.Offset = int64(arm64.REGZERO)
   517  		p.To.Type = obj.TYPE_MEM
   518  		p.To.Reg = v.Args[0].Reg()
   519  		ssagen.AddAux(&p.To, v)
   520  	case ssa.OpARM64BFI,
   521  		ssa.OpARM64BFXIL:
   522  		p := s.Prog(v.Op.Asm())
   523  		p.From.Type = obj.TYPE_CONST
   524  		p.From.Offset = v.AuxInt >> 8
   525  		p.SetFrom3Const(v.AuxInt & 0xff)
   526  		p.Reg = v.Args[1].Reg()
   527  		p.To.Type = obj.TYPE_REG
   528  		p.To.Reg = v.Reg()
   529  	case ssa.OpARM64SBFIZ,
   530  		ssa.OpARM64SBFX,
   531  		ssa.OpARM64UBFIZ,
   532  		ssa.OpARM64UBFX:
   533  		p := s.Prog(v.Op.Asm())
   534  		p.From.Type = obj.TYPE_CONST
   535  		p.From.Offset = v.AuxInt >> 8
   536  		p.SetFrom3Const(v.AuxInt & 0xff)
   537  		p.Reg = v.Args[0].Reg()
   538  		p.To.Type = obj.TYPE_REG
   539  		p.To.Reg = v.Reg()
   540  	case ssa.OpARM64LoweredMuluhilo:
   541  		r0 := v.Args[0].Reg()
   542  		r1 := v.Args[1].Reg()
   543  		p := s.Prog(arm64.AUMULH)
   544  		p.From.Type = obj.TYPE_REG
   545  		p.From.Reg = r1
   546  		p.Reg = r0
   547  		p.To.Type = obj.TYPE_REG
   548  		p.To.Reg = v.Reg0()
   549  		p1 := s.Prog(arm64.AMUL)
   550  		p1.From.Type = obj.TYPE_REG
   551  		p1.From.Reg = r1
   552  		p1.Reg = r0
   553  		p1.To.Type = obj.TYPE_REG
   554  		p1.To.Reg = v.Reg1()
   555  	case ssa.OpARM64LoweredAtomicExchange64,
   556  		ssa.OpARM64LoweredAtomicExchange32:
   557  		// LDAXR	(Rarg0), Rout
   558  		// STLXR	Rarg1, (Rarg0), Rtmp
   559  		// CBNZ		Rtmp, -2(PC)
   560  		ld := arm64.ALDAXR
   561  		st := arm64.ASTLXR
   562  		if v.Op == ssa.OpARM64LoweredAtomicExchange32 {
   563  			ld = arm64.ALDAXRW
   564  			st = arm64.ASTLXRW
   565  		}
   566  		r0 := v.Args[0].Reg()
   567  		r1 := v.Args[1].Reg()
   568  		out := v.Reg0()
   569  		p := s.Prog(ld)
   570  		p.From.Type = obj.TYPE_MEM
   571  		p.From.Reg = r0
   572  		p.To.Type = obj.TYPE_REG
   573  		p.To.Reg = out
   574  		p1 := s.Prog(st)
   575  		p1.From.Type = obj.TYPE_REG
   576  		p1.From.Reg = r1
   577  		p1.To.Type = obj.TYPE_MEM
   578  		p1.To.Reg = r0
   579  		p1.RegTo2 = arm64.REGTMP
   580  		p2 := s.Prog(arm64.ACBNZ)
   581  		p2.From.Type = obj.TYPE_REG
   582  		p2.From.Reg = arm64.REGTMP
   583  		p2.To.Type = obj.TYPE_BRANCH
   584  		p2.To.SetTarget(p)
   585  	case ssa.OpARM64LoweredAtomicExchange64Variant,
   586  		ssa.OpARM64LoweredAtomicExchange32Variant:
   587  		swap := arm64.ASWPALD
   588  		if v.Op == ssa.OpARM64LoweredAtomicExchange32Variant {
   589  			swap = arm64.ASWPALW
   590  		}
   591  		r0 := v.Args[0].Reg()
   592  		r1 := v.Args[1].Reg()
   593  		out := v.Reg0()
   594  
   595  		// SWPALD	Rarg1, (Rarg0), Rout
   596  		p := s.Prog(swap)
   597  		p.From.Type = obj.TYPE_REG
   598  		p.From.Reg = r1
   599  		p.To.Type = obj.TYPE_MEM
   600  		p.To.Reg = r0
   601  		p.RegTo2 = out
   602  
   603  	case ssa.OpARM64LoweredAtomicAdd64,
   604  		ssa.OpARM64LoweredAtomicAdd32:
   605  		// LDAXR	(Rarg0), Rout
   606  		// ADD		Rarg1, Rout
   607  		// STLXR	Rout, (Rarg0), Rtmp
   608  		// CBNZ		Rtmp, -3(PC)
   609  		ld := arm64.ALDAXR
   610  		st := arm64.ASTLXR
   611  		if v.Op == ssa.OpARM64LoweredAtomicAdd32 {
   612  			ld = arm64.ALDAXRW
   613  			st = arm64.ASTLXRW
   614  		}
   615  		r0 := v.Args[0].Reg()
   616  		r1 := v.Args[1].Reg()
   617  		out := v.Reg0()
   618  		p := s.Prog(ld)
   619  		p.From.Type = obj.TYPE_MEM
   620  		p.From.Reg = r0
   621  		p.To.Type = obj.TYPE_REG
   622  		p.To.Reg = out
   623  		p1 := s.Prog(arm64.AADD)
   624  		p1.From.Type = obj.TYPE_REG
   625  		p1.From.Reg = r1
   626  		p1.To.Type = obj.TYPE_REG
   627  		p1.To.Reg = out
   628  		p2 := s.Prog(st)
   629  		p2.From.Type = obj.TYPE_REG
   630  		p2.From.Reg = out
   631  		p2.To.Type = obj.TYPE_MEM
   632  		p2.To.Reg = r0
   633  		p2.RegTo2 = arm64.REGTMP
   634  		p3 := s.Prog(arm64.ACBNZ)
   635  		p3.From.Type = obj.TYPE_REG
   636  		p3.From.Reg = arm64.REGTMP
   637  		p3.To.Type = obj.TYPE_BRANCH
   638  		p3.To.SetTarget(p)
   639  	case ssa.OpARM64LoweredAtomicAdd64Variant,
   640  		ssa.OpARM64LoweredAtomicAdd32Variant:
   641  		// LDADDAL	Rarg1, (Rarg0), Rout
   642  		// ADD		Rarg1, Rout
   643  		op := arm64.ALDADDALD
   644  		if v.Op == ssa.OpARM64LoweredAtomicAdd32Variant {
   645  			op = arm64.ALDADDALW
   646  		}
   647  		r0 := v.Args[0].Reg()
   648  		r1 := v.Args[1].Reg()
   649  		out := v.Reg0()
   650  		p := s.Prog(op)
   651  		p.From.Type = obj.TYPE_REG
   652  		p.From.Reg = r1
   653  		p.To.Type = obj.TYPE_MEM
   654  		p.To.Reg = r0
   655  		p.RegTo2 = out
   656  		p1 := s.Prog(arm64.AADD)
   657  		p1.From.Type = obj.TYPE_REG
   658  		p1.From.Reg = r1
   659  		p1.To.Type = obj.TYPE_REG
   660  		p1.To.Reg = out
   661  	case ssa.OpARM64LoweredAtomicCas64,
   662  		ssa.OpARM64LoweredAtomicCas32:
   663  		// LDAXR	(Rarg0), Rtmp
   664  		// CMP		Rarg1, Rtmp
   665  		// BNE		3(PC)
   666  		// STLXR	Rarg2, (Rarg0), Rtmp
   667  		// CBNZ		Rtmp, -4(PC)
   668  		// CSET		EQ, Rout
   669  		ld := arm64.ALDAXR
   670  		st := arm64.ASTLXR
   671  		cmp := arm64.ACMP
   672  		if v.Op == ssa.OpARM64LoweredAtomicCas32 {
   673  			ld = arm64.ALDAXRW
   674  			st = arm64.ASTLXRW
   675  			cmp = arm64.ACMPW
   676  		}
   677  		r0 := v.Args[0].Reg()
   678  		r1 := v.Args[1].Reg()
   679  		r2 := v.Args[2].Reg()
   680  		out := v.Reg0()
   681  		p := s.Prog(ld)
   682  		p.From.Type = obj.TYPE_MEM
   683  		p.From.Reg = r0
   684  		p.To.Type = obj.TYPE_REG
   685  		p.To.Reg = arm64.REGTMP
   686  		p1 := s.Prog(cmp)
   687  		p1.From.Type = obj.TYPE_REG
   688  		p1.From.Reg = r1
   689  		p1.Reg = arm64.REGTMP
   690  		p2 := s.Prog(arm64.ABNE)
   691  		p2.To.Type = obj.TYPE_BRANCH
   692  		p3 := s.Prog(st)
   693  		p3.From.Type = obj.TYPE_REG
   694  		p3.From.Reg = r2
   695  		p3.To.Type = obj.TYPE_MEM
   696  		p3.To.Reg = r0
   697  		p3.RegTo2 = arm64.REGTMP
   698  		p4 := s.Prog(arm64.ACBNZ)
   699  		p4.From.Type = obj.TYPE_REG
   700  		p4.From.Reg = arm64.REGTMP
   701  		p4.To.Type = obj.TYPE_BRANCH
   702  		p4.To.SetTarget(p)
   703  		p5 := s.Prog(arm64.ACSET)
   704  		p5.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   705  		p5.From.Reg = arm64.COND_EQ
   706  		p5.To.Type = obj.TYPE_REG
   707  		p5.To.Reg = out
   708  		p2.To.SetTarget(p5)
   709  	case ssa.OpARM64LoweredAtomicCas64Variant,
   710  		ssa.OpARM64LoweredAtomicCas32Variant:
   711  		// Rarg0: ptr
   712  		// Rarg1: old
   713  		// Rarg2: new
   714  		// MOV  	Rarg1, Rtmp
   715  		// CASAL	Rtmp, (Rarg0), Rarg2
   716  		// CMP  	Rarg1, Rtmp
   717  		// CSET 	EQ, Rout
   718  		cas := arm64.ACASALD
   719  		cmp := arm64.ACMP
   720  		mov := arm64.AMOVD
   721  		if v.Op == ssa.OpARM64LoweredAtomicCas32Variant {
   722  			cas = arm64.ACASALW
   723  			cmp = arm64.ACMPW
   724  			mov = arm64.AMOVW
   725  		}
   726  		r0 := v.Args[0].Reg()
   727  		r1 := v.Args[1].Reg()
   728  		r2 := v.Args[2].Reg()
   729  		out := v.Reg0()
   730  
   731  		// MOV  	Rarg1, Rtmp
   732  		p := s.Prog(mov)
   733  		p.From.Type = obj.TYPE_REG
   734  		p.From.Reg = r1
   735  		p.To.Type = obj.TYPE_REG
   736  		p.To.Reg = arm64.REGTMP
   737  
   738  		// CASAL	Rtmp, (Rarg0), Rarg2
   739  		p1 := s.Prog(cas)
   740  		p1.From.Type = obj.TYPE_REG
   741  		p1.From.Reg = arm64.REGTMP
   742  		p1.To.Type = obj.TYPE_MEM
   743  		p1.To.Reg = r0
   744  		p1.RegTo2 = r2
   745  
   746  		// CMP  	Rarg1, Rtmp
   747  		p2 := s.Prog(cmp)
   748  		p2.From.Type = obj.TYPE_REG
   749  		p2.From.Reg = r1
   750  		p2.Reg = arm64.REGTMP
   751  
   752  		// CSET 	EQ, Rout
   753  		p3 := s.Prog(arm64.ACSET)
   754  		p3.From.Type = obj.TYPE_REG
   755  		p3.From.Reg = arm64.COND_EQ
   756  		p3.To.Type = obj.TYPE_REG
   757  		p3.To.Reg = out
   758  
   759  	case ssa.OpARM64LoweredAtomicAnd8,
   760  		ssa.OpARM64LoweredAtomicAnd32,
   761  		ssa.OpARM64LoweredAtomicOr8,
   762  		ssa.OpARM64LoweredAtomicOr32:
   763  		// LDAXRB/LDAXRW (Rarg0), Rout
   764  		// AND/OR	Rarg1, Rout
   765  		// STLXRB/STLXRB Rout, (Rarg0), Rtmp
   766  		// CBNZ		Rtmp, -3(PC)
   767  		ld := arm64.ALDAXRB
   768  		st := arm64.ASTLXRB
   769  		if v.Op == ssa.OpARM64LoweredAtomicAnd32 || v.Op == ssa.OpARM64LoweredAtomicOr32 {
   770  			ld = arm64.ALDAXRW
   771  			st = arm64.ASTLXRW
   772  		}
   773  		r0 := v.Args[0].Reg()
   774  		r1 := v.Args[1].Reg()
   775  		out := v.Reg0()
   776  		p := s.Prog(ld)
   777  		p.From.Type = obj.TYPE_MEM
   778  		p.From.Reg = r0
   779  		p.To.Type = obj.TYPE_REG
   780  		p.To.Reg = out
   781  		p1 := s.Prog(v.Op.Asm())
   782  		p1.From.Type = obj.TYPE_REG
   783  		p1.From.Reg = r1
   784  		p1.To.Type = obj.TYPE_REG
   785  		p1.To.Reg = out
   786  		p2 := s.Prog(st)
   787  		p2.From.Type = obj.TYPE_REG
   788  		p2.From.Reg = out
   789  		p2.To.Type = obj.TYPE_MEM
   790  		p2.To.Reg = r0
   791  		p2.RegTo2 = arm64.REGTMP
   792  		p3 := s.Prog(arm64.ACBNZ)
   793  		p3.From.Type = obj.TYPE_REG
   794  		p3.From.Reg = arm64.REGTMP
   795  		p3.To.Type = obj.TYPE_BRANCH
   796  		p3.To.SetTarget(p)
   797  	case ssa.OpARM64LoweredAtomicAnd8Variant,
   798  		ssa.OpARM64LoweredAtomicAnd32Variant:
   799  		atomic_clear := arm64.ALDCLRALW
   800  		if v.Op == ssa.OpARM64LoweredAtomicAnd8Variant {
   801  			atomic_clear = arm64.ALDCLRALB
   802  		}
   803  		r0 := v.Args[0].Reg()
   804  		r1 := v.Args[1].Reg()
   805  		out := v.Reg0()
   806  
   807  		// MNV       Rarg1 Rtemp
   808  		p := s.Prog(arm64.AMVN)
   809  		p.From.Type = obj.TYPE_REG
   810  		p.From.Reg = r1
   811  		p.To.Type = obj.TYPE_REG
   812  		p.To.Reg = arm64.REGTMP
   813  
   814  		// LDCLRALW  Rtemp, (Rarg0), Rout
   815  		p1 := s.Prog(atomic_clear)
   816  		p1.From.Type = obj.TYPE_REG
   817  		p1.From.Reg = arm64.REGTMP
   818  		p1.To.Type = obj.TYPE_MEM
   819  		p1.To.Reg = r0
   820  		p1.RegTo2 = out
   821  
   822  		// AND       Rarg1, Rout
   823  		p2 := s.Prog(arm64.AAND)
   824  		p2.From.Type = obj.TYPE_REG
   825  		p2.From.Reg = r1
   826  		p2.To.Type = obj.TYPE_REG
   827  		p2.To.Reg = out
   828  
   829  	case ssa.OpARM64LoweredAtomicOr8Variant,
   830  		ssa.OpARM64LoweredAtomicOr32Variant:
   831  		atomic_or := arm64.ALDORALW
   832  		if v.Op == ssa.OpARM64LoweredAtomicOr8Variant {
   833  			atomic_or = arm64.ALDORALB
   834  		}
   835  		r0 := v.Args[0].Reg()
   836  		r1 := v.Args[1].Reg()
   837  		out := v.Reg0()
   838  
   839  		// LDORALW  Rarg1, (Rarg0), Rout
   840  		p := s.Prog(atomic_or)
   841  		p.From.Type = obj.TYPE_REG
   842  		p.From.Reg = r1
   843  		p.To.Type = obj.TYPE_MEM
   844  		p.To.Reg = r0
   845  		p.RegTo2 = out
   846  
   847  		// ORR       Rarg1, Rout
   848  		p2 := s.Prog(arm64.AORR)
   849  		p2.From.Type = obj.TYPE_REG
   850  		p2.From.Reg = r1
   851  		p2.To.Type = obj.TYPE_REG
   852  		p2.To.Reg = out
   853  
   854  	case ssa.OpARM64MOVBreg,
   855  		ssa.OpARM64MOVBUreg,
   856  		ssa.OpARM64MOVHreg,
   857  		ssa.OpARM64MOVHUreg,
   858  		ssa.OpARM64MOVWreg,
   859  		ssa.OpARM64MOVWUreg:
   860  		a := v.Args[0]
   861  		for a.Op == ssa.OpCopy || a.Op == ssa.OpARM64MOVDreg {
   862  			a = a.Args[0]
   863  		}
   864  		if a.Op == ssa.OpLoadReg {
   865  			t := a.Type
   866  			switch {
   867  			case v.Op == ssa.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(),
   868  				v.Op == ssa.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
   869  				v.Op == ssa.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(),
   870  				v.Op == ssa.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
   871  				v.Op == ssa.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(),
   872  				v.Op == ssa.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned():
   873  				// arg is a proper-typed load, already zero/sign-extended, don't extend again
   874  				if v.Reg() == v.Args[0].Reg() {
   875  					return
   876  				}
   877  				p := s.Prog(arm64.AMOVD)
   878  				p.From.Type = obj.TYPE_REG
   879  				p.From.Reg = v.Args[0].Reg()
   880  				p.To.Type = obj.TYPE_REG
   881  				p.To.Reg = v.Reg()
   882  				return
   883  			default:
   884  			}
   885  		}
   886  		fallthrough
   887  	case ssa.OpARM64MVN,
   888  		ssa.OpARM64NEG,
   889  		ssa.OpARM64FABSD,
   890  		ssa.OpARM64FMOVDfpgp,
   891  		ssa.OpARM64FMOVDgpfp,
   892  		ssa.OpARM64FMOVSfpgp,
   893  		ssa.OpARM64FMOVSgpfp,
   894  		ssa.OpARM64FNEGS,
   895  		ssa.OpARM64FNEGD,
   896  		ssa.OpARM64FSQRTS,
   897  		ssa.OpARM64FSQRTD,
   898  		ssa.OpARM64FCVTZSSW,
   899  		ssa.OpARM64FCVTZSDW,
   900  		ssa.OpARM64FCVTZUSW,
   901  		ssa.OpARM64FCVTZUDW,
   902  		ssa.OpARM64FCVTZSS,
   903  		ssa.OpARM64FCVTZSD,
   904  		ssa.OpARM64FCVTZUS,
   905  		ssa.OpARM64FCVTZUD,
   906  		ssa.OpARM64SCVTFWS,
   907  		ssa.OpARM64SCVTFWD,
   908  		ssa.OpARM64SCVTFS,
   909  		ssa.OpARM64SCVTFD,
   910  		ssa.OpARM64UCVTFWS,
   911  		ssa.OpARM64UCVTFWD,
   912  		ssa.OpARM64UCVTFS,
   913  		ssa.OpARM64UCVTFD,
   914  		ssa.OpARM64FCVTSD,
   915  		ssa.OpARM64FCVTDS,
   916  		ssa.OpARM64REV,
   917  		ssa.OpARM64REVW,
   918  		ssa.OpARM64REV16,
   919  		ssa.OpARM64REV16W,
   920  		ssa.OpARM64RBIT,
   921  		ssa.OpARM64RBITW,
   922  		ssa.OpARM64CLZ,
   923  		ssa.OpARM64CLZW,
   924  		ssa.OpARM64FRINTAD,
   925  		ssa.OpARM64FRINTMD,
   926  		ssa.OpARM64FRINTND,
   927  		ssa.OpARM64FRINTPD,
   928  		ssa.OpARM64FRINTZD:
   929  		p := s.Prog(v.Op.Asm())
   930  		p.From.Type = obj.TYPE_REG
   931  		p.From.Reg = v.Args[0].Reg()
   932  		p.To.Type = obj.TYPE_REG
   933  		p.To.Reg = v.Reg()
   934  	case ssa.OpARM64LoweredRound32F, ssa.OpARM64LoweredRound64F:
   935  		// input is already rounded
   936  	case ssa.OpARM64VCNT:
   937  		p := s.Prog(v.Op.Asm())
   938  		p.From.Type = obj.TYPE_REG
   939  		p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
   940  		p.To.Type = obj.TYPE_REG
   941  		p.To.Reg = (v.Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
   942  	case ssa.OpARM64VUADDLV:
   943  		p := s.Prog(v.Op.Asm())
   944  		p.From.Type = obj.TYPE_REG
   945  		p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
   946  		p.To.Type = obj.TYPE_REG
   947  		p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
   948  	case ssa.OpARM64CSEL, ssa.OpARM64CSEL0:
   949  		r1 := int16(arm64.REGZERO)
   950  		if v.Op != ssa.OpARM64CSEL0 {
   951  			r1 = v.Args[1].Reg()
   952  		}
   953  		p := s.Prog(v.Op.Asm())
   954  		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   955  		p.From.Reg = condBits[ssa.Op(v.AuxInt)]
   956  		p.Reg = v.Args[0].Reg()
   957  		p.SetFrom3Reg(r1)
   958  		p.To.Type = obj.TYPE_REG
   959  		p.To.Reg = v.Reg()
   960  	case ssa.OpARM64CSINC, ssa.OpARM64CSINV, ssa.OpARM64CSNEG:
   961  		p := s.Prog(v.Op.Asm())
   962  		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   963  		p.From.Reg = condBits[ssa.Op(v.AuxInt)]
   964  		p.Reg = v.Args[0].Reg()
   965  		p.SetFrom3Reg(v.Args[1].Reg())
   966  		p.To.Type = obj.TYPE_REG
   967  		p.To.Reg = v.Reg()
   968  	case ssa.OpARM64CSETM:
   969  		p := s.Prog(arm64.ACSETM)
   970  		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   971  		p.From.Reg = condBits[ssa.Op(v.AuxInt)]
   972  		p.To.Type = obj.TYPE_REG
   973  		p.To.Reg = v.Reg()
   974  	case ssa.OpARM64DUFFZERO:
   975  		// runtime.duffzero expects start address in R20
   976  		p := s.Prog(obj.ADUFFZERO)
   977  		p.To.Type = obj.TYPE_MEM
   978  		p.To.Name = obj.NAME_EXTERN
   979  		p.To.Sym = ir.Syms.Duffzero
   980  		p.To.Offset = v.AuxInt
   981  	case ssa.OpARM64LoweredZero:
   982  		// STP.P	(ZR,ZR), 16(R16)
   983  		// CMP	Rarg1, R16
   984  		// BLE	-2(PC)
   985  		// arg1 is the address of the last 16-byte unit to zero
   986  		p := s.Prog(arm64.ASTP)
   987  		p.Scond = arm64.C_XPOST
   988  		p.From.Type = obj.TYPE_REGREG
   989  		p.From.Reg = arm64.REGZERO
   990  		p.From.Offset = int64(arm64.REGZERO)
   991  		p.To.Type = obj.TYPE_MEM
   992  		p.To.Reg = arm64.REG_R16
   993  		p.To.Offset = 16
   994  		p2 := s.Prog(arm64.ACMP)
   995  		p2.From.Type = obj.TYPE_REG
   996  		p2.From.Reg = v.Args[1].Reg()
   997  		p2.Reg = arm64.REG_R16
   998  		p3 := s.Prog(arm64.ABLE)
   999  		p3.To.Type = obj.TYPE_BRANCH
  1000  		p3.To.SetTarget(p)
  1001  	case ssa.OpARM64DUFFCOPY:
  1002  		p := s.Prog(obj.ADUFFCOPY)
  1003  		p.To.Type = obj.TYPE_MEM
  1004  		p.To.Name = obj.NAME_EXTERN
  1005  		p.To.Sym = ir.Syms.Duffcopy
  1006  		p.To.Offset = v.AuxInt
  1007  	case ssa.OpARM64LoweredMove:
  1008  		// MOVD.P	8(R16), Rtmp
  1009  		// MOVD.P	Rtmp, 8(R17)
  1010  		// CMP	Rarg2, R16
  1011  		// BLE	-3(PC)
  1012  		// arg2 is the address of the last element of src
  1013  		p := s.Prog(arm64.AMOVD)
  1014  		p.Scond = arm64.C_XPOST
  1015  		p.From.Type = obj.TYPE_MEM
  1016  		p.From.Reg = arm64.REG_R16
  1017  		p.From.Offset = 8
  1018  		p.To.Type = obj.TYPE_REG
  1019  		p.To.Reg = arm64.REGTMP
  1020  		p2 := s.Prog(arm64.AMOVD)
  1021  		p2.Scond = arm64.C_XPOST
  1022  		p2.From.Type = obj.TYPE_REG
  1023  		p2.From.Reg = arm64.REGTMP
  1024  		p2.To.Type = obj.TYPE_MEM
  1025  		p2.To.Reg = arm64.REG_R17
  1026  		p2.To.Offset = 8
  1027  		p3 := s.Prog(arm64.ACMP)
  1028  		p3.From.Type = obj.TYPE_REG
  1029  		p3.From.Reg = v.Args[2].Reg()
  1030  		p3.Reg = arm64.REG_R16
  1031  		p4 := s.Prog(arm64.ABLE)
  1032  		p4.To.Type = obj.TYPE_BRANCH
  1033  		p4.To.SetTarget(p)
  1034  	case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter:
  1035  		s.Call(v)
  1036  	case ssa.OpARM64LoweredWB:
  1037  		p := s.Prog(obj.ACALL)
  1038  		p.To.Type = obj.TYPE_MEM
  1039  		p.To.Name = obj.NAME_EXTERN
  1040  		p.To.Sym = v.Aux.(*obj.LSym)
  1041  	case ssa.OpARM64LoweredPanicBoundsA, ssa.OpARM64LoweredPanicBoundsB, ssa.OpARM64LoweredPanicBoundsC:
  1042  		p := s.Prog(obj.ACALL)
  1043  		p.To.Type = obj.TYPE_MEM
  1044  		p.To.Name = obj.NAME_EXTERN
  1045  		p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt]
  1046  		s.UseArgs(16) // space used in callee args area by assembly stubs
  1047  	case ssa.OpARM64LoweredNilCheck:
  1048  		// Issue a load which will fault if arg is nil.
  1049  		p := s.Prog(arm64.AMOVB)
  1050  		p.From.Type = obj.TYPE_MEM
  1051  		p.From.Reg = v.Args[0].Reg()
  1052  		ssagen.AddAux(&p.From, v)
  1053  		p.To.Type = obj.TYPE_REG
  1054  		p.To.Reg = arm64.REGTMP
  1055  		if logopt.Enabled() {
  1056  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1057  		}
  1058  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Line==1 in generated wrappers
  1059  			base.WarnfAt(v.Pos, "generated nil check")
  1060  		}
  1061  	case ssa.OpARM64Equal,
  1062  		ssa.OpARM64NotEqual,
  1063  		ssa.OpARM64LessThan,
  1064  		ssa.OpARM64LessEqual,
  1065  		ssa.OpARM64GreaterThan,
  1066  		ssa.OpARM64GreaterEqual,
  1067  		ssa.OpARM64LessThanU,
  1068  		ssa.OpARM64LessEqualU,
  1069  		ssa.OpARM64GreaterThanU,
  1070  		ssa.OpARM64GreaterEqualU,
  1071  		ssa.OpARM64LessThanF,
  1072  		ssa.OpARM64LessEqualF,
  1073  		ssa.OpARM64GreaterThanF,
  1074  		ssa.OpARM64GreaterEqualF,
  1075  		ssa.OpARM64NotLessThanF,
  1076  		ssa.OpARM64NotLessEqualF,
  1077  		ssa.OpARM64NotGreaterThanF,
  1078  		ssa.OpARM64NotGreaterEqualF:
  1079  		// generate boolean values using CSET
  1080  		p := s.Prog(arm64.ACSET)
  1081  		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
  1082  		p.From.Reg = condBits[v.Op]
  1083  		p.To.Type = obj.TYPE_REG
  1084  		p.To.Reg = v.Reg()
  1085  	case ssa.OpARM64LoweredGetClosurePtr:
  1086  		// Closure pointer is R26 (arm64.REGCTXT).
  1087  		ssagen.CheckLoweredGetClosurePtr(v)
  1088  	case ssa.OpARM64LoweredGetCallerSP:
  1089  		// caller's SP is FixedFrameSize below the address of the first arg
  1090  		p := s.Prog(arm64.AMOVD)
  1091  		p.From.Type = obj.TYPE_ADDR
  1092  		p.From.Offset = -base.Ctxt.FixedFrameSize()
  1093  		p.From.Name = obj.NAME_PARAM
  1094  		p.To.Type = obj.TYPE_REG
  1095  		p.To.Reg = v.Reg()
  1096  	case ssa.OpARM64LoweredGetCallerPC:
  1097  		p := s.Prog(obj.AGETCALLERPC)
  1098  		p.To.Type = obj.TYPE_REG
  1099  		p.To.Reg = v.Reg()
  1100  	case ssa.OpARM64FlagConstant:
  1101  		v.Fatalf("FlagConstant op should never make it to codegen %v", v.LongString())
  1102  	case ssa.OpARM64InvertFlags:
  1103  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  1104  	case ssa.OpClobber, ssa.OpClobberReg:
  1105  		// TODO: implement for clobberdead experiment. Nop is ok for now.
  1106  	default:
  1107  		v.Fatalf("genValue not implemented: %s", v.LongString())
  1108  	}
  1109  }
  1110  
  1111  var condBits = map[ssa.Op]int16{
  1112  	ssa.OpARM64Equal:         arm64.COND_EQ,
  1113  	ssa.OpARM64NotEqual:      arm64.COND_NE,
  1114  	ssa.OpARM64LessThan:      arm64.COND_LT,
  1115  	ssa.OpARM64LessThanU:     arm64.COND_LO,
  1116  	ssa.OpARM64LessEqual:     arm64.COND_LE,
  1117  	ssa.OpARM64LessEqualU:    arm64.COND_LS,
  1118  	ssa.OpARM64GreaterThan:   arm64.COND_GT,
  1119  	ssa.OpARM64GreaterThanU:  arm64.COND_HI,
  1120  	ssa.OpARM64GreaterEqual:  arm64.COND_GE,
  1121  	ssa.OpARM64GreaterEqualU: arm64.COND_HS,
  1122  	ssa.OpARM64LessThanF:     arm64.COND_MI, // Less than
  1123  	ssa.OpARM64LessEqualF:    arm64.COND_LS, // Less than or equal to
  1124  	ssa.OpARM64GreaterThanF:  arm64.COND_GT, // Greater than
  1125  	ssa.OpARM64GreaterEqualF: arm64.COND_GE, // Greater than or equal to
  1126  
  1127  	// The following condition codes have unordered to handle comparisons related to NaN.
  1128  	ssa.OpARM64NotLessThanF:     arm64.COND_PL, // Greater than, equal to, or unordered
  1129  	ssa.OpARM64NotLessEqualF:    arm64.COND_HI, // Greater than or unordered
  1130  	ssa.OpARM64NotGreaterThanF:  arm64.COND_LE, // Less than, equal to or unordered
  1131  	ssa.OpARM64NotGreaterEqualF: arm64.COND_LT, // Less than or unordered
  1132  }
  1133  
  1134  var blockJump = map[ssa.BlockKind]struct {
  1135  	asm, invasm obj.As
  1136  }{
  1137  	ssa.BlockARM64EQ:     {arm64.ABEQ, arm64.ABNE},
  1138  	ssa.BlockARM64NE:     {arm64.ABNE, arm64.ABEQ},
  1139  	ssa.BlockARM64LT:     {arm64.ABLT, arm64.ABGE},
  1140  	ssa.BlockARM64GE:     {arm64.ABGE, arm64.ABLT},
  1141  	ssa.BlockARM64LE:     {arm64.ABLE, arm64.ABGT},
  1142  	ssa.BlockARM64GT:     {arm64.ABGT, arm64.ABLE},
  1143  	ssa.BlockARM64ULT:    {arm64.ABLO, arm64.ABHS},
  1144  	ssa.BlockARM64UGE:    {arm64.ABHS, arm64.ABLO},
  1145  	ssa.BlockARM64UGT:    {arm64.ABHI, arm64.ABLS},
  1146  	ssa.BlockARM64ULE:    {arm64.ABLS, arm64.ABHI},
  1147  	ssa.BlockARM64Z:      {arm64.ACBZ, arm64.ACBNZ},
  1148  	ssa.BlockARM64NZ:     {arm64.ACBNZ, arm64.ACBZ},
  1149  	ssa.BlockARM64ZW:     {arm64.ACBZW, arm64.ACBNZW},
  1150  	ssa.BlockARM64NZW:    {arm64.ACBNZW, arm64.ACBZW},
  1151  	ssa.BlockARM64TBZ:    {arm64.ATBZ, arm64.ATBNZ},
  1152  	ssa.BlockARM64TBNZ:   {arm64.ATBNZ, arm64.ATBZ},
  1153  	ssa.BlockARM64FLT:    {arm64.ABMI, arm64.ABPL},
  1154  	ssa.BlockARM64FGE:    {arm64.ABGE, arm64.ABLT},
  1155  	ssa.BlockARM64FLE:    {arm64.ABLS, arm64.ABHI},
  1156  	ssa.BlockARM64FGT:    {arm64.ABGT, arm64.ABLE},
  1157  	ssa.BlockARM64LTnoov: {arm64.ABMI, arm64.ABPL},
  1158  	ssa.BlockARM64GEnoov: {arm64.ABPL, arm64.ABMI},
  1159  }
  1160  
  1161  // To model a 'LEnoov' ('<=' without overflow checking) branching
  1162  var leJumps = [2][2]ssagen.IndexJump{
  1163  	{{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}}, // next == b.Succs[0]
  1164  	{{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}}, // next == b.Succs[1]
  1165  }
  1166  
  1167  // To model a 'GTnoov' ('>' without overflow checking) branching
  1168  var gtJumps = [2][2]ssagen.IndexJump{
  1169  	{{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}}, // next == b.Succs[0]
  1170  	{{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}}, // next == b.Succs[1]
  1171  }
  1172  
  1173  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  1174  	switch b.Kind {
  1175  	case ssa.BlockPlain:
  1176  		if b.Succs[0].Block() != next {
  1177  			p := s.Prog(obj.AJMP)
  1178  			p.To.Type = obj.TYPE_BRANCH
  1179  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1180  		}
  1181  
  1182  	case ssa.BlockDefer:
  1183  		// defer returns in R0:
  1184  		// 0 if we should continue executing
  1185  		// 1 if we should jump to deferreturn call
  1186  		p := s.Prog(arm64.ACMP)
  1187  		p.From.Type = obj.TYPE_CONST
  1188  		p.From.Offset = 0
  1189  		p.Reg = arm64.REG_R0
  1190  		p = s.Prog(arm64.ABNE)
  1191  		p.To.Type = obj.TYPE_BRANCH
  1192  		s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()})
  1193  		if b.Succs[0].Block() != next {
  1194  			p := s.Prog(obj.AJMP)
  1195  			p.To.Type = obj.TYPE_BRANCH
  1196  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1197  		}
  1198  
  1199  	case ssa.BlockExit:
  1200  
  1201  	case ssa.BlockRet:
  1202  		s.Prog(obj.ARET)
  1203  
  1204  	case ssa.BlockRetJmp:
  1205  		p := s.Prog(obj.ARET)
  1206  		p.To.Type = obj.TYPE_MEM
  1207  		p.To.Name = obj.NAME_EXTERN
  1208  		p.To.Sym = b.Aux.(*obj.LSym)
  1209  
  1210  	case ssa.BlockARM64EQ, ssa.BlockARM64NE,
  1211  		ssa.BlockARM64LT, ssa.BlockARM64GE,
  1212  		ssa.BlockARM64LE, ssa.BlockARM64GT,
  1213  		ssa.BlockARM64ULT, ssa.BlockARM64UGT,
  1214  		ssa.BlockARM64ULE, ssa.BlockARM64UGE,
  1215  		ssa.BlockARM64Z, ssa.BlockARM64NZ,
  1216  		ssa.BlockARM64ZW, ssa.BlockARM64NZW,
  1217  		ssa.BlockARM64FLT, ssa.BlockARM64FGE,
  1218  		ssa.BlockARM64FLE, ssa.BlockARM64FGT,
  1219  		ssa.BlockARM64LTnoov, ssa.BlockARM64GEnoov:
  1220  		jmp := blockJump[b.Kind]
  1221  		var p *obj.Prog
  1222  		switch next {
  1223  		case b.Succs[0].Block():
  1224  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  1225  		case b.Succs[1].Block():
  1226  			p = s.Br(jmp.asm, b.Succs[0].Block())
  1227  		default:
  1228  			if b.Likely != ssa.BranchUnlikely {
  1229  				p = s.Br(jmp.asm, b.Succs[0].Block())
  1230  				s.Br(obj.AJMP, b.Succs[1].Block())
  1231  			} else {
  1232  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  1233  				s.Br(obj.AJMP, b.Succs[0].Block())
  1234  			}
  1235  		}
  1236  		if !b.Controls[0].Type.IsFlags() {
  1237  			p.From.Type = obj.TYPE_REG
  1238  			p.From.Reg = b.Controls[0].Reg()
  1239  		}
  1240  	case ssa.BlockARM64TBZ, ssa.BlockARM64TBNZ:
  1241  		jmp := blockJump[b.Kind]
  1242  		var p *obj.Prog
  1243  		switch next {
  1244  		case b.Succs[0].Block():
  1245  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  1246  		case b.Succs[1].Block():
  1247  			p = s.Br(jmp.asm, b.Succs[0].Block())
  1248  		default:
  1249  			if b.Likely != ssa.BranchUnlikely {
  1250  				p = s.Br(jmp.asm, b.Succs[0].Block())
  1251  				s.Br(obj.AJMP, b.Succs[1].Block())
  1252  			} else {
  1253  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  1254  				s.Br(obj.AJMP, b.Succs[0].Block())
  1255  			}
  1256  		}
  1257  		p.From.Offset = b.AuxInt
  1258  		p.From.Type = obj.TYPE_CONST
  1259  		p.Reg = b.Controls[0].Reg()
  1260  
  1261  	case ssa.BlockARM64LEnoov:
  1262  		s.CombJump(b, next, &leJumps)
  1263  	case ssa.BlockARM64GTnoov:
  1264  		s.CombJump(b, next, &gtJumps)
  1265  	default:
  1266  		b.Fatalf("branch not implemented: %s", b.LongString())
  1267  	}
  1268  }
  1269  

View as plain text