atomic_arm.go

Documentation: runtime/internal/atomic

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build arm
     6  // +build arm
     7  
     8  package atomic
     9  
    10  import (
    11  	"internal/cpu"
    12  	"unsafe"
    13  )
    14  
    15  // Export some functions via linkname to assembly in sync/atomic.
    16  //go:linkname Xchg
    17  //go:linkname Xchguintptr
    18  
    19  type spinlock struct {
    20  	v uint32
    21  }
    22  
    23  //go:nosplit
    24  func (l *spinlock) lock() {
    25  	for {
    26  		if Cas(&l.v, 0, 1) {
    27  			return
    28  		}
    29  	}
    30  }
    31  
    32  //go:nosplit
    33  func (l *spinlock) unlock() {
    34  	Store(&l.v, 0)
    35  }
    36  
    37  var locktab [57]struct {
    38  	l   spinlock
    39  	pad [cpu.CacheLinePadSize - unsafe.Sizeof(spinlock{})]byte
    40  }
    41  
    42  func addrLock(addr *uint64) *spinlock {
    43  	return &locktab[(uintptr(unsafe.Pointer(addr))>>3)%uintptr(len(locktab))].l
    44  }
    45  
    46  // Atomic add and return new value.
    47  //go:nosplit
    48  func Xadd(val *uint32, delta int32) uint32 {
    49  	for {
    50  		oval := *val
    51  		nval := oval + uint32(delta)
    52  		if Cas(val, oval, nval) {
    53  			return nval
    54  		}
    55  	}
    56  }
    57  
    58  //go:noescape
    59  func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
    60  
    61  //go:nosplit
    62  func Xchg(addr *uint32, v uint32) uint32 {
    63  	for {
    64  		old := *addr
    65  		if Cas(addr, old, v) {
    66  			return old
    67  		}
    68  	}
    69  }
    70  
    71  //go:nosplit
    72  func Xchguintptr(addr *uintptr, v uintptr) uintptr {
    73  	return uintptr(Xchg((*uint32)(unsafe.Pointer(addr)), uint32(v)))
    74  }
    75  
    76  // Not noescape -- it installs a pointer to addr.
    77  func StorepNoWB(addr unsafe.Pointer, v unsafe.Pointer)
    78  
    79  //go:noescape
    80  func Store(addr *uint32, v uint32)
    81  
    82  //go:noescape
    83  func StoreRel(addr *uint32, v uint32)
    84  
    85  //go:noescape
    86  func StoreReluintptr(addr *uintptr, v uintptr)
    87  
    88  //go:nosplit
    89  func goCas64(addr *uint64, old, new uint64) bool {
    90  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
    91  		*(*int)(nil) = 0 // crash on unaligned uint64
    92  	}
    93  	_ = *addr // if nil, fault before taking the lock
    94  	var ok bool
    95  	addrLock(addr).lock()
    96  	if *addr == old {
    97  		*addr = new
    98  		ok = true
    99  	}
   100  	addrLock(addr).unlock()
   101  	return ok
   102  }
   103  
   104  //go:nosplit
   105  func goXadd64(addr *uint64, delta int64) uint64 {
   106  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   107  		*(*int)(nil) = 0 // crash on unaligned uint64
   108  	}
   109  	_ = *addr // if nil, fault before taking the lock
   110  	var r uint64
   111  	addrLock(addr).lock()
   112  	r = *addr + uint64(delta)
   113  	*addr = r
   114  	addrLock(addr).unlock()
   115  	return r
   116  }
   117  
   118  //go:nosplit
   119  func goXchg64(addr *uint64, v uint64) uint64 {
   120  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   121  		*(*int)(nil) = 0 // crash on unaligned uint64
   122  	}
   123  	_ = *addr // if nil, fault before taking the lock
   124  	var r uint64
   125  	addrLock(addr).lock()
   126  	r = *addr
   127  	*addr = v
   128  	addrLock(addr).unlock()
   129  	return r
   130  }
   131  
   132  //go:nosplit
   133  func goLoad64(addr *uint64) uint64 {
   134  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   135  		*(*int)(nil) = 0 // crash on unaligned uint64
   136  	}
   137  	_ = *addr // if nil, fault before taking the lock
   138  	var r uint64
   139  	addrLock(addr).lock()
   140  	r = *addr
   141  	addrLock(addr).unlock()
   142  	return r
   143  }
   144  
   145  //go:nosplit
   146  func goStore64(addr *uint64, v uint64) {
   147  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   148  		*(*int)(nil) = 0 // crash on unaligned uint64
   149  	}
   150  	_ = *addr // if nil, fault before taking the lock
   151  	addrLock(addr).lock()
   152  	*addr = v
   153  	addrLock(addr).unlock()
   154  }
   155  
   156  //go:nosplit
   157  func Or8(addr *uint8, v uint8) {
   158  	// Align down to 4 bytes and use 32-bit CAS.
   159  	uaddr := uintptr(unsafe.Pointer(addr))
   160  	addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3))
   161  	word := uint32(v) << ((uaddr & 3) * 8) // little endian
   162  	for {
   163  		old := *addr32
   164  		if Cas(addr32, old, old|word) {
   165  			return
   166  		}
   167  	}
   168  }
   169  
   170  //go:nosplit
   171  func And8(addr *uint8, v uint8) {
   172  	// Align down to 4 bytes and use 32-bit CAS.
   173  	uaddr := uintptr(unsafe.Pointer(addr))
   174  	addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3))
   175  	word := uint32(v) << ((uaddr & 3) * 8)    // little endian
   176  	mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian
   177  	word |= ^mask
   178  	for {
   179  		old := *addr32
   180  		if Cas(addr32, old, old&word) {
   181  			return
   182  		}
   183  	}
   184  }
   185  
   186  //go:nosplit
   187  func Or(addr *uint32, v uint32) {
   188  	for {
   189  		old := *addr
   190  		if Cas(addr, old, old|v) {
   191  			return
   192  		}
   193  	}
   194  }
   195  
   196  //go:nosplit
   197  func And(addr *uint32, v uint32) {
   198  	for {
   199  		old := *addr
   200  		if Cas(addr, old, old&v) {
   201  			return
   202  		}
   203  	}
   204  }
   205  
   206  //go:nosplit
   207  func armcas(ptr *uint32, old, new uint32) bool
   208  
   209  //go:noescape
   210  func Load(addr *uint32) uint32
   211  
   212  // NO go:noescape annotation; *addr escapes if result escapes (#31525)
   213  func Loadp(addr unsafe.Pointer) unsafe.Pointer
   214  
   215  //go:noescape
   216  func Load8(addr *uint8) uint8
   217  
   218  //go:noescape
   219  func LoadAcq(addr *uint32) uint32
   220  
   221  //go:noescape
   222  func LoadAcquintptr(ptr *uintptr) uintptr
   223  
   224  //go:noescape
   225  func Cas64(addr *uint64, old, new uint64) bool
   226  
   227  //go:noescape
   228  func CasRel(addr *uint32, old, new uint32) bool
   229  
   230  //go:noescape
   231  func Xadd64(addr *uint64, delta int64) uint64
   232  
   233  //go:noescape
   234  func Xchg64(addr *uint64, v uint64) uint64
   235  
   236  //go:noescape
   237  func Load64(addr *uint64) uint64
   238  
   239  //go:noescape
   240  func Store8(addr *uint8, v uint8)
   241  
   242  //go:noescape
   243  func Store64(addr *uint64, v uint64)
   244
View as plain text