1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "textflag.h"
6 #include "funcdata.h"
7
8 // bool armcas(int32 *val, int32 old, int32 new)
9 // Atomically:
10 // if(*val == old){
11 // *val = new;
12 // return 1;
13 // }else
14 // return 0;
15 //
16 // To implement ·cas in sys_$GOOS_arm.s
17 // using the native instructions, use:
18 //
19 // TEXT ·cas(SB),NOSPLIT,$0
20 // B ·armcas(SB)
21 //
22 TEXT ·armcas(SB),NOSPLIT,$0-13
23 MOVW ptr+0(FP), R1
24 MOVW old+4(FP), R2
25 MOVW new+8(FP), R3
26 casl:
27 LDREX (R1), R0
28 CMP R0, R2
29 BNE casfail
30
31 MOVB runtime·goarm(SB), R8
32 CMP $7, R8
33 BLT 2(PC)
34 DMB MB_ISHST
35
36 STREX R3, (R1), R0
37 CMP $0, R0
38 BNE casl
39 MOVW $1, R0
40
41 CMP $7, R8
42 BLT 2(PC)
43 DMB MB_ISH
44
45 MOVB R0, ret+12(FP)
46 RET
47 casfail:
48 MOVW $0, R0
49 MOVB R0, ret+12(FP)
50 RET
51
52 // stubs
53
54 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
55 B ·Load(SB)
56
57 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
58 B ·Load(SB)
59
60 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
61 B ·Load(SB)
62
63 TEXT ·Casint32(SB),NOSPLIT,$0-13
64 B ·Cas(SB)
65
66 TEXT ·Casint64(SB),NOSPLIT,$-4-21
67 B ·Cas64(SB)
68
69 TEXT ·Casuintptr(SB),NOSPLIT,$0-13
70 B ·Cas(SB)
71
72 TEXT ·Casp1(SB),NOSPLIT,$0-13
73 B ·Cas(SB)
74
75 TEXT ·CasRel(SB),NOSPLIT,$0-13
76 B ·Cas(SB)
77
78 TEXT ·Loadint32(SB),NOSPLIT,$0-8
79 B ·Load(SB)
80
81 TEXT ·Loadint64(SB),NOSPLIT,$-4-12
82 B ·Load64(SB)
83
84 TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
85 B ·Load(SB)
86
87 TEXT ·Loaduint(SB),NOSPLIT,$0-8
88 B ·Load(SB)
89
90 TEXT ·Storeint32(SB),NOSPLIT,$0-8
91 B ·Store(SB)
92
93 TEXT ·Storeint64(SB),NOSPLIT,$0-12
94 B ·Store64(SB)
95
96 TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
97 B ·Store(SB)
98
99 TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
100 B ·Store(SB)
101
102 TEXT ·StoreRel(SB),NOSPLIT,$0-8
103 B ·Store(SB)
104
105 TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
106 B ·Store(SB)
107
108 TEXT ·Xaddint32(SB),NOSPLIT,$0-12
109 B ·Xadd(SB)
110
111 TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
112 B ·Xadd64(SB)
113
114 TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
115 B ·Xadd(SB)
116
117 TEXT ·Xchgint32(SB),NOSPLIT,$0-12
118 B ·Xchg(SB)
119
120 TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
121 B ·Xchg64(SB)
122
123 // 64-bit atomics
124 // The native ARM implementations use LDREXD/STREXD, which are
125 // available on ARMv6k or later. We use them only on ARMv7.
126 // On older ARM, we use Go implementations which simulate 64-bit
127 // atomics with locks.
128 TEXT armCas64<>(SB),NOSPLIT,$0-21
129 // addr is already in R1
130 MOVW old_lo+4(FP), R2
131 MOVW old_hi+8(FP), R3
132 MOVW new_lo+12(FP), R4
133 MOVW new_hi+16(FP), R5
134 cas64loop:
135 LDREXD (R1), R6 // loads R6 and R7
136 CMP R2, R6
137 BNE cas64fail
138 CMP R3, R7
139 BNE cas64fail
140
141 DMB MB_ISHST
142
143 STREXD R4, (R1), R0 // stores R4 and R5
144 CMP $0, R0
145 BNE cas64loop
146 MOVW $1, R0
147
148 DMB MB_ISH
149
150 MOVBU R0, swapped+20(FP)
151 RET
152 cas64fail:
153 MOVW $0, R0
154 MOVBU R0, swapped+20(FP)
155 RET
156
157 TEXT armXadd64<>(SB),NOSPLIT,$0-20
158 // addr is already in R1
159 MOVW delta_lo+4(FP), R2
160 MOVW delta_hi+8(FP), R3
161
162 add64loop:
163 LDREXD (R1), R4 // loads R4 and R5
164 ADD.S R2, R4
165 ADC R3, R5
166
167 DMB MB_ISHST
168
169 STREXD R4, (R1), R0 // stores R4 and R5
170 CMP $0, R0
171 BNE add64loop
172
173 DMB MB_ISH
174
175 MOVW R4, new_lo+12(FP)
176 MOVW R5, new_hi+16(FP)
177 RET
178
179 TEXT armXchg64<>(SB),NOSPLIT,$0-20
180 // addr is already in R1
181 MOVW new_lo+4(FP), R2
182 MOVW new_hi+8(FP), R3
183
184 swap64loop:
185 LDREXD (R1), R4 // loads R4 and R5
186
187 DMB MB_ISHST
188
189 STREXD R2, (R1), R0 // stores R2 and R3
190 CMP $0, R0
191 BNE swap64loop
192
193 DMB MB_ISH
194
195 MOVW R4, old_lo+12(FP)
196 MOVW R5, old_hi+16(FP)
197 RET
198
199 TEXT armLoad64<>(SB),NOSPLIT,$0-12
200 // addr is already in R1
201
202 LDREXD (R1), R2 // loads R2 and R3
203 DMB MB_ISH
204
205 MOVW R2, val_lo+4(FP)
206 MOVW R3, val_hi+8(FP)
207 RET
208
209 TEXT armStore64<>(SB),NOSPLIT,$0-12
210 // addr is already in R1
211 MOVW val_lo+4(FP), R2
212 MOVW val_hi+8(FP), R3
213
214 store64loop:
215 LDREXD (R1), R4 // loads R4 and R5
216
217 DMB MB_ISHST
218
219 STREXD R2, (R1), R0 // stores R2 and R3
220 CMP $0, R0
221 BNE store64loop
222
223 DMB MB_ISH
224 RET
225
226 // The following functions all panic if their address argument isn't
227 // 8-byte aligned. Since we're calling back into Go code to do this,
228 // we have to cooperate with stack unwinding. In the normal case, the
229 // functions tail-call into the appropriate implementation, which
230 // means they must not open a frame. Hence, when they go down the
231 // panic path, at that point they push the LR to create a real frame
232 // (they don't need to pop it because panic won't return).
233
234 TEXT ·Cas64(SB),NOSPLIT,$-4-21
235 NO_LOCAL_POINTERS
236 MOVW addr+0(FP), R1
237 // make unaligned atomic access panic
238 AND.S $7, R1, R2
239 BEQ 3(PC)
240 MOVW.W R14, -4(R13) // prepare a real frame
241 BL ·panicUnaligned(SB)
242
243 MOVB runtime·goarm(SB), R11
244 CMP $7, R11
245 BLT 2(PC)
246 JMP armCas64<>(SB)
247 JMP ·goCas64(SB)
248
249 TEXT ·Xadd64(SB),NOSPLIT,$-4-20
250 NO_LOCAL_POINTERS
251 MOVW addr+0(FP), R1
252 // make unaligned atomic access panic
253 AND.S $7, R1, R2
254 BEQ 3(PC)
255 MOVW.W R14, -4(R13) // prepare a real frame
256 BL ·panicUnaligned(SB)
257
258 MOVB runtime·goarm(SB), R11
259 CMP $7, R11
260 BLT 2(PC)
261 JMP armXadd64<>(SB)
262 JMP ·goXadd64(SB)
263
264 TEXT ·Xchg64(SB),NOSPLIT,$-4-20
265 NO_LOCAL_POINTERS
266 MOVW addr+0(FP), R1
267 // make unaligned atomic access panic
268 AND.S $7, R1, R2
269 BEQ 3(PC)
270 MOVW.W R14, -4(R13) // prepare a real frame
271 BL ·panicUnaligned(SB)
272
273 MOVB runtime·goarm(SB), R11
274 CMP $7, R11
275 BLT 2(PC)
276 JMP armXchg64<>(SB)
277 JMP ·goXchg64(SB)
278
279 TEXT ·Load64(SB),NOSPLIT,$-4-12
280 NO_LOCAL_POINTERS
281 MOVW addr+0(FP), R1
282 // make unaligned atomic access panic
283 AND.S $7, R1, R2
284 BEQ 3(PC)
285 MOVW.W R14, -4(R13) // prepare a real frame
286 BL ·panicUnaligned(SB)
287
288 MOVB runtime·goarm(SB), R11
289 CMP $7, R11
290 BLT 2(PC)
291 JMP armLoad64<>(SB)
292 JMP ·goLoad64(SB)
293
294 TEXT ·Store64(SB),NOSPLIT,$-4-12
295 NO_LOCAL_POINTERS
296 MOVW addr+0(FP), R1
297 // make unaligned atomic access panic
298 AND.S $7, R1, R2
299 BEQ 3(PC)
300 MOVW.W R14, -4(R13) // prepare a real frame
301 BL ·panicUnaligned(SB)
302
303 MOVB runtime·goarm(SB), R11
304 CMP $7, R11
305 BLT 2(PC)
306 JMP armStore64<>(SB)
307 JMP ·goStore64(SB)
308
View as plain text