Source file src/runtime/export_test.go

Documentation: runtime

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Export guts for testing.
     6  
     7  package runtime
     8  
     9  import (
    10  	"runtime/internal/atomic"
    11  	"runtime/internal/sys"
    12  	"unsafe"
    13  )
    14  
    15  var Fadd64 = fadd64
    16  var Fsub64 = fsub64
    17  var Fmul64 = fmul64
    18  var Fdiv64 = fdiv64
    19  var F64to32 = f64to32
    20  var F32to64 = f32to64
    21  var Fcmp64 = fcmp64
    22  var Fintto64 = fintto64
    23  var F64toint = f64toint
    24  
    25  var Entersyscall = entersyscall
    26  var Exitsyscall = exitsyscall
    27  var LockedOSThread = lockedOSThread
    28  var Xadduintptr = atomic.Xadduintptr
    29  
    30  var FuncPC = funcPC
    31  
    32  var Fastlog2 = fastlog2
    33  
    34  var Atoi = atoi
    35  var Atoi32 = atoi32
    36  
    37  var Nanotime = nanotime
    38  var NetpollBreak = netpollBreak
    39  var Usleep = usleep
    40  
    41  var PhysPageSize = physPageSize
    42  var PhysHugePageSize = physHugePageSize
    43  
    44  var NetpollGenericInit = netpollGenericInit
    45  
    46  var Memmove = memmove
    47  var MemclrNoHeapPointers = memclrNoHeapPointers
    48  
    49  var LockPartialOrder = lockPartialOrder
    50  
    51  type LockRank lockRank
    52  
    53  func (l LockRank) String() string {
    54  	return lockRank(l).String()
    55  }
    56  
    57  const PreemptMSupported = preemptMSupported
    58  
    59  type LFNode struct {
    60  	Next    uint64
    61  	Pushcnt uintptr
    62  }
    63  
    64  func LFStackPush(head *uint64, node *LFNode) {
    65  	(*lfstack)(head).push((*lfnode)(unsafe.Pointer(node)))
    66  }
    67  
    68  func LFStackPop(head *uint64) *LFNode {
    69  	return (*LFNode)(unsafe.Pointer((*lfstack)(head).pop()))
    70  }
    71  
    72  func Netpoll(delta int64) {
    73  	systemstack(func() {
    74  		netpoll(delta)
    75  	})
    76  }
    77  
    78  func GCMask(x interface{}) (ret []byte) {
    79  	systemstack(func() {
    80  		ret = getgcmask(x)
    81  	})
    82  	return
    83  }
    84  
    85  func RunSchedLocalQueueTest() {
    86  	_p_ := new(p)
    87  	gs := make([]g, len(_p_.runq))
    88  	for i := 0; i < len(_p_.runq); i++ {
    89  		if g, _ := runqget(_p_); g != nil {
    90  			throw("runq is not empty initially")
    91  		}
    92  		for j := 0; j < i; j++ {
    93  			runqput(_p_, &gs[i], false)
    94  		}
    95  		for j := 0; j < i; j++ {
    96  			if g, _ := runqget(_p_); g != &gs[i] {
    97  				print("bad element at iter ", i, "/", j, "\n")
    98  				throw("bad element")
    99  			}
   100  		}
   101  		if g, _ := runqget(_p_); g != nil {
   102  			throw("runq is not empty afterwards")
   103  		}
   104  	}
   105  }
   106  
   107  func RunSchedLocalQueueStealTest() {
   108  	p1 := new(p)
   109  	p2 := new(p)
   110  	gs := make([]g, len(p1.runq))
   111  	for i := 0; i < len(p1.runq); i++ {
   112  		for j := 0; j < i; j++ {
   113  			gs[j].sig = 0
   114  			runqput(p1, &gs[j], false)
   115  		}
   116  		gp := runqsteal(p2, p1, true)
   117  		s := 0
   118  		if gp != nil {
   119  			s++
   120  			gp.sig++
   121  		}
   122  		for {
   123  			gp, _ = runqget(p2)
   124  			if gp == nil {
   125  				break
   126  			}
   127  			s++
   128  			gp.sig++
   129  		}
   130  		for {
   131  			gp, _ = runqget(p1)
   132  			if gp == nil {
   133  				break
   134  			}
   135  			gp.sig++
   136  		}
   137  		for j := 0; j < i; j++ {
   138  			if gs[j].sig != 1 {
   139  				print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
   140  				throw("bad element")
   141  			}
   142  		}
   143  		if s != i/2 && s != i/2+1 {
   144  			print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
   145  			throw("bad steal")
   146  		}
   147  	}
   148  }
   149  
   150  // Temporary to enable register ABI bringup.
   151  // TODO(register args): convert back to local variables in RunSchedLocalQueueEmptyTest that
   152  // get passed to the "go" stmts there.
   153  var RunSchedLocalQueueEmptyState struct {
   154  	done  chan bool
   155  	ready *uint32
   156  	p     *p
   157  }
   158  
   159  func RunSchedLocalQueueEmptyTest(iters int) {
   160  	// Test that runq is not spuriously reported as empty.
   161  	// Runq emptiness affects scheduling decisions and spurious emptiness
   162  	// can lead to underutilization (both runnable Gs and idle Ps coexist
   163  	// for arbitrary long time).
   164  	done := make(chan bool, 1)
   165  	RunSchedLocalQueueEmptyState.done = done
   166  	p := new(p)
   167  	RunSchedLocalQueueEmptyState.p = p
   168  	gs := make([]g, 2)
   169  	ready := new(uint32)
   170  	RunSchedLocalQueueEmptyState.ready = ready
   171  	for i := 0; i < iters; i++ {
   172  		*ready = 0
   173  		next0 := (i & 1) == 0
   174  		next1 := (i & 2) == 0
   175  		runqput(p, &gs[0], next0)
   176  		go func() {
   177  			for atomic.Xadd(RunSchedLocalQueueEmptyState.ready, 1); atomic.Load(RunSchedLocalQueueEmptyState.ready) != 2; {
   178  			}
   179  			if runqempty(RunSchedLocalQueueEmptyState.p) {
   180  				//println("next:", next0, next1)
   181  				throw("queue is empty")
   182  			}
   183  			RunSchedLocalQueueEmptyState.done <- true
   184  		}()
   185  		for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
   186  		}
   187  		runqput(p, &gs[1], next1)
   188  		runqget(p)
   189  		<-done
   190  		runqget(p)
   191  	}
   192  }
   193  
   194  var (
   195  	StringHash = stringHash
   196  	BytesHash  = bytesHash
   197  	Int32Hash  = int32Hash
   198  	Int64Hash  = int64Hash
   199  	MemHash    = memhash
   200  	MemHash32  = memhash32
   201  	MemHash64  = memhash64
   202  	EfaceHash  = efaceHash
   203  	IfaceHash  = ifaceHash
   204  )
   205  
   206  var UseAeshash = &useAeshash
   207  
   208  func MemclrBytes(b []byte) {
   209  	s := (*slice)(unsafe.Pointer(&b))
   210  	memclrNoHeapPointers(s.array, uintptr(s.len))
   211  }
   212  
   213  var HashLoad = &hashLoad
   214  
   215  // entry point for testing
   216  func GostringW(w []uint16) (s string) {
   217  	systemstack(func() {
   218  		s = gostringw(&w[0])
   219  	})
   220  	return
   221  }
   222  
   223  var Open = open
   224  var Close = closefd
   225  var Read = read
   226  var Write = write
   227  
   228  func Envs() []string     { return envs }
   229  func SetEnvs(e []string) { envs = e }
   230  
   231  var BigEndian = sys.BigEndian
   232  
   233  // For benchmarking.
   234  
   235  func BenchSetType(n int, x interface{}) {
   236  	e := *efaceOf(&x)
   237  	t := e._type
   238  	var size uintptr
   239  	var p unsafe.Pointer
   240  	switch t.kind & kindMask {
   241  	case kindPtr:
   242  		t = (*ptrtype)(unsafe.Pointer(t)).elem
   243  		size = t.size
   244  		p = e.data
   245  	case kindSlice:
   246  		slice := *(*struct {
   247  			ptr      unsafe.Pointer
   248  			len, cap uintptr
   249  		})(e.data)
   250  		t = (*slicetype)(unsafe.Pointer(t)).elem
   251  		size = t.size * slice.len
   252  		p = slice.ptr
   253  	}
   254  	allocSize := roundupsize(size)
   255  	systemstack(func() {
   256  		for i := 0; i < n; i++ {
   257  			heapBitsSetType(uintptr(p), allocSize, size, t)
   258  		}
   259  	})
   260  }
   261  
   262  const PtrSize = sys.PtrSize
   263  
   264  var ForceGCPeriod = &forcegcperiod
   265  
   266  // SetTracebackEnv is like runtime/debug.SetTraceback, but it raises
   267  // the "environment" traceback level, so later calls to
   268  // debug.SetTraceback (e.g., from testing timeouts) can't lower it.
   269  func SetTracebackEnv(level string) {
   270  	setTraceback(level)
   271  	traceback_env = traceback_cache
   272  }
   273  
   274  var ReadUnaligned32 = readUnaligned32
   275  var ReadUnaligned64 = readUnaligned64
   276  
   277  func CountPagesInUse() (pagesInUse, counted uintptr) {
   278  	stopTheWorld("CountPagesInUse")
   279  
   280  	pagesInUse = uintptr(mheap_.pagesInUse)
   281  
   282  	for _, s := range mheap_.allspans {
   283  		if s.state.get() == mSpanInUse {
   284  			counted += s.npages
   285  		}
   286  	}
   287  
   288  	startTheWorld()
   289  
   290  	return
   291  }
   292  
   293  func Fastrand() uint32          { return fastrand() }
   294  func Fastrandn(n uint32) uint32 { return fastrandn(n) }
   295  
   296  type ProfBuf profBuf
   297  
   298  func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf {
   299  	return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags))
   300  }
   301  
   302  func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) {
   303  	(*profBuf)(p).write(tag, now, hdr, stk)
   304  }
   305  
   306  const (
   307  	ProfBufBlocking    = profBufBlocking
   308  	ProfBufNonBlocking = profBufNonBlocking
   309  )
   310  
   311  func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) {
   312  	return (*profBuf)(p).read(profBufReadMode(mode))
   313  }
   314  
   315  func (p *ProfBuf) Close() {
   316  	(*profBuf)(p).close()
   317  }
   318  
   319  func ReadMetricsSlow(memStats *MemStats, samplesp unsafe.Pointer, len, cap int) {
   320  	stopTheWorld("ReadMetricsSlow")
   321  
   322  	// Initialize the metrics beforehand because this could
   323  	// allocate and skew the stats.
   324  	semacquire(&metricsSema)
   325  	initMetrics()
   326  	semrelease(&metricsSema)
   327  
   328  	systemstack(func() {
   329  		// Read memstats first. It's going to flush
   330  		// the mcaches which readMetrics does not do, so
   331  		// going the other way around may result in
   332  		// inconsistent statistics.
   333  		readmemstats_m(memStats)
   334  	})
   335  
   336  	// Read metrics off the system stack.
   337  	//
   338  	// The only part of readMetrics that could allocate
   339  	// and skew the stats is initMetrics.
   340  	readMetrics(samplesp, len, cap)
   341  
   342  	startTheWorld()
   343  }
   344  
   345  // ReadMemStatsSlow returns both the runtime-computed MemStats and
   346  // MemStats accumulated by scanning the heap.
   347  func ReadMemStatsSlow() (base, slow MemStats) {
   348  	stopTheWorld("ReadMemStatsSlow")
   349  
   350  	// Run on the system stack to avoid stack growth allocation.
   351  	systemstack(func() {
   352  		// Make sure stats don't change.
   353  		getg().m.mallocing++
   354  
   355  		readmemstats_m(&base)
   356  
   357  		// Initialize slow from base and zero the fields we're
   358  		// recomputing.
   359  		slow = base
   360  		slow.Alloc = 0
   361  		slow.TotalAlloc = 0
   362  		slow.Mallocs = 0
   363  		slow.Frees = 0
   364  		slow.HeapReleased = 0
   365  		var bySize [_NumSizeClasses]struct {
   366  			Mallocs, Frees uint64
   367  		}
   368  
   369  		// Add up current allocations in spans.
   370  		for _, s := range mheap_.allspans {
   371  			if s.state.get() != mSpanInUse {
   372  				continue
   373  			}
   374  			if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 {
   375  				slow.Mallocs++
   376  				slow.Alloc += uint64(s.elemsize)
   377  			} else {
   378  				slow.Mallocs += uint64(s.allocCount)
   379  				slow.Alloc += uint64(s.allocCount) * uint64(s.elemsize)
   380  				bySize[sizeclass].Mallocs += uint64(s.allocCount)
   381  			}
   382  		}
   383  
   384  		// Add in frees by just reading the stats for those directly.
   385  		var m heapStatsDelta
   386  		memstats.heapStats.unsafeRead(&m)
   387  
   388  		// Collect per-sizeclass free stats.
   389  		var smallFree uint64
   390  		for i := 0; i < _NumSizeClasses; i++ {
   391  			slow.Frees += uint64(m.smallFreeCount[i])
   392  			bySize[i].Frees += uint64(m.smallFreeCount[i])
   393  			bySize[i].Mallocs += uint64(m.smallFreeCount[i])
   394  			smallFree += uint64(m.smallFreeCount[i]) * uint64(class_to_size[i])
   395  		}
   396  		slow.Frees += uint64(m.tinyAllocCount) + uint64(m.largeFreeCount)
   397  		slow.Mallocs += slow.Frees
   398  
   399  		slow.TotalAlloc = slow.Alloc + uint64(m.largeFree) + smallFree
   400  
   401  		for i := range slow.BySize {
   402  			slow.BySize[i].Mallocs = bySize[i].Mallocs
   403  			slow.BySize[i].Frees = bySize[i].Frees
   404  		}
   405  
   406  		for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
   407  			chunk := mheap_.pages.tryChunkOf(i)
   408  			if chunk == nil {
   409  				continue
   410  			}
   411  			pg := chunk.scavenged.popcntRange(0, pallocChunkPages)
   412  			slow.HeapReleased += uint64(pg) * pageSize
   413  		}
   414  		for _, p := range allp {
   415  			pg := sys.OnesCount64(p.pcache.scav)
   416  			slow.HeapReleased += uint64(pg) * pageSize
   417  		}
   418  
   419  		getg().m.mallocing--
   420  	})
   421  
   422  	startTheWorld()
   423  	return
   424  }
   425  
   426  // BlockOnSystemStack switches to the system stack, prints "x\n" to
   427  // stderr, and blocks in a stack containing
   428  // "runtime.blockOnSystemStackInternal".
   429  func BlockOnSystemStack() {
   430  	systemstack(blockOnSystemStackInternal)
   431  }
   432  
   433  func blockOnSystemStackInternal() {
   434  	print("x\n")
   435  	lock(&deadlock)
   436  	lock(&deadlock)
   437  }
   438  
   439  type RWMutex struct {
   440  	rw rwmutex
   441  }
   442  
   443  func (rw *RWMutex) RLock() {
   444  	rw.rw.rlock()
   445  }
   446  
   447  func (rw *RWMutex) RUnlock() {
   448  	rw.rw.runlock()
   449  }
   450  
   451  func (rw *RWMutex) Lock() {
   452  	rw.rw.lock()
   453  }
   454  
   455  func (rw *RWMutex) Unlock() {
   456  	rw.rw.unlock()
   457  }
   458  
   459  const RuntimeHmapSize = unsafe.Sizeof(hmap{})
   460  
   461  func MapBucketsCount(m map[int]int) int {
   462  	h := *(**hmap)(unsafe.Pointer(&m))
   463  	return 1 << h.B
   464  }
   465  
   466  func MapBucketsPointerIsNil(m map[int]int) bool {
   467  	h := *(**hmap)(unsafe.Pointer(&m))
   468  	return h.buckets == nil
   469  }
   470  
   471  func LockOSCounts() (external, internal uint32) {
   472  	g := getg()
   473  	if g.m.lockedExt+g.m.lockedInt == 0 {
   474  		if g.lockedm != 0 {
   475  			panic("lockedm on non-locked goroutine")
   476  		}
   477  	} else {
   478  		if g.lockedm == 0 {
   479  			panic("nil lockedm on locked goroutine")
   480  		}
   481  	}
   482  	return g.m.lockedExt, g.m.lockedInt
   483  }
   484  
   485  //go:noinline
   486  func TracebackSystemstack(stk []uintptr, i int) int {
   487  	if i == 0 {
   488  		pc, sp := getcallerpc(), getcallersp()
   489  		return gentraceback(pc, sp, 0, getg(), 0, &stk[0], len(stk), nil, nil, _TraceJumpStack)
   490  	}
   491  	n := 0
   492  	systemstack(func() {
   493  		n = TracebackSystemstack(stk, i-1)
   494  	})
   495  	return n
   496  }
   497  
   498  func KeepNArenaHints(n int) {
   499  	hint := mheap_.arenaHints
   500  	for i := 1; i < n; i++ {
   501  		hint = hint.next
   502  		if hint == nil {
   503  			return
   504  		}
   505  	}
   506  	hint.next = nil
   507  }
   508  
   509  // MapNextArenaHint reserves a page at the next arena growth hint,
   510  // preventing the arena from growing there, and returns the range of
   511  // addresses that are no longer viable.
   512  func MapNextArenaHint() (start, end uintptr) {
   513  	hint := mheap_.arenaHints
   514  	addr := hint.addr
   515  	if hint.down {
   516  		start, end = addr-heapArenaBytes, addr
   517  		addr -= physPageSize
   518  	} else {
   519  		start, end = addr, addr+heapArenaBytes
   520  	}
   521  	sysReserve(unsafe.Pointer(addr), physPageSize)
   522  	return
   523  }
   524  
   525  func GetNextArenaHint() uintptr {
   526  	return mheap_.arenaHints.addr
   527  }
   528  
   529  type G = g
   530  
   531  type Sudog = sudog
   532  
   533  func Getg() *G {
   534  	return getg()
   535  }
   536  
   537  //go:noinline
   538  func PanicForTesting(b []byte, i int) byte {
   539  	return unexportedPanicForTesting(b, i)
   540  }
   541  
   542  //go:noinline
   543  func unexportedPanicForTesting(b []byte, i int) byte {
   544  	return b[i]
   545  }
   546  
   547  func G0StackOverflow() {
   548  	systemstack(func() {
   549  		stackOverflow(nil)
   550  	})
   551  }
   552  
   553  func stackOverflow(x *byte) {
   554  	var buf [256]byte
   555  	stackOverflow(&buf[0])
   556  }
   557  
   558  func MapTombstoneCheck(m map[int]int) {
   559  	// Make sure emptyOne and emptyRest are distributed correctly.
   560  	// We should have a series of filled and emptyOne cells, followed by
   561  	// a series of emptyRest cells.
   562  	h := *(**hmap)(unsafe.Pointer(&m))
   563  	i := interface{}(m)
   564  	t := *(**maptype)(unsafe.Pointer(&i))
   565  
   566  	for x := 0; x < 1<<h.B; x++ {
   567  		b0 := (*bmap)(add(h.buckets, uintptr(x)*uintptr(t.bucketsize)))
   568  		n := 0
   569  		for b := b0; b != nil; b = b.overflow(t) {
   570  			for i := 0; i < bucketCnt; i++ {
   571  				if b.tophash[i] != emptyRest {
   572  					n++
   573  				}
   574  			}
   575  		}
   576  		k := 0
   577  		for b := b0; b != nil; b = b.overflow(t) {
   578  			for i := 0; i < bucketCnt; i++ {
   579  				if k < n && b.tophash[i] == emptyRest {
   580  					panic("early emptyRest")
   581  				}
   582  				if k >= n && b.tophash[i] != emptyRest {
   583  					panic("late non-emptyRest")
   584  				}
   585  				if k == n-1 && b.tophash[i] == emptyOne {
   586  					panic("last non-emptyRest entry is emptyOne")
   587  				}
   588  				k++
   589  			}
   590  		}
   591  	}
   592  }
   593  
   594  func RunGetgThreadSwitchTest() {
   595  	// Test that getg works correctly with thread switch.
   596  	// With gccgo, if we generate getg inlined, the backend
   597  	// may cache the address of the TLS variable, which
   598  	// will become invalid after a thread switch. This test
   599  	// checks that the bad caching doesn't happen.
   600  
   601  	ch := make(chan int)
   602  	go func(ch chan int) {
   603  		ch <- 5
   604  		LockOSThread()
   605  	}(ch)
   606  
   607  	g1 := getg()
   608  
   609  	// Block on a receive. This is likely to get us a thread
   610  	// switch. If we yield to the sender goroutine, it will
   611  	// lock the thread, forcing us to resume on a different
   612  	// thread.
   613  	<-ch
   614  
   615  	g2 := getg()
   616  	if g1 != g2 {
   617  		panic("g1 != g2")
   618  	}
   619  
   620  	// Also test getg after some control flow, as the
   621  	// backend is sensitive to control flow.
   622  	g3 := getg()
   623  	if g1 != g3 {
   624  		panic("g1 != g3")
   625  	}
   626  }
   627  
   628  const (
   629  	PageSize         = pageSize
   630  	PallocChunkPages = pallocChunkPages
   631  	PageAlloc64Bit   = pageAlloc64Bit
   632  	PallocSumBytes   = pallocSumBytes
   633  )
   634  
   635  // Expose pallocSum for testing.
   636  type PallocSum pallocSum
   637  
   638  func PackPallocSum(start, max, end uint) PallocSum { return PallocSum(packPallocSum(start, max, end)) }
   639  func (m PallocSum) Start() uint                    { return pallocSum(m).start() }
   640  func (m PallocSum) Max() uint                      { return pallocSum(m).max() }
   641  func (m PallocSum) End() uint                      { return pallocSum(m).end() }
   642  
   643  // Expose pallocBits for testing.
   644  type PallocBits pallocBits
   645  
   646  func (b *PallocBits) Find(npages uintptr, searchIdx uint) (uint, uint) {
   647  	return (*pallocBits)(b).find(npages, searchIdx)
   648  }
   649  func (b *PallocBits) AllocRange(i, n uint)       { (*pallocBits)(b).allocRange(i, n) }
   650  func (b *PallocBits) Free(i, n uint)             { (*pallocBits)(b).free(i, n) }
   651  func (b *PallocBits) Summarize() PallocSum       { return PallocSum((*pallocBits)(b).summarize()) }
   652  func (b *PallocBits) PopcntRange(i, n uint) uint { return (*pageBits)(b).popcntRange(i, n) }
   653  
   654  // SummarizeSlow is a slow but more obviously correct implementation
   655  // of (*pallocBits).summarize. Used for testing.
   656  func SummarizeSlow(b *PallocBits) PallocSum {
   657  	var start, max, end uint
   658  
   659  	const N = uint(len(b)) * 64
   660  	for start < N && (*pageBits)(b).get(start) == 0 {
   661  		start++
   662  	}
   663  	for end < N && (*pageBits)(b).get(N-end-1) == 0 {
   664  		end++
   665  	}
   666  	run := uint(0)
   667  	for i := uint(0); i < N; i++ {
   668  		if (*pageBits)(b).get(i) == 0 {
   669  			run++
   670  		} else {
   671  			run = 0
   672  		}
   673  		if run > max {
   674  			max = run
   675  		}
   676  	}
   677  	return PackPallocSum(start, max, end)
   678  }
   679  
   680  // Expose non-trivial helpers for testing.
   681  func FindBitRange64(c uint64, n uint) uint { return findBitRange64(c, n) }
   682  
   683  // Given two PallocBits, returns a set of bit ranges where
   684  // they differ.
   685  func DiffPallocBits(a, b *PallocBits) []BitRange {
   686  	ba := (*pageBits)(a)
   687  	bb := (*pageBits)(b)
   688  
   689  	var d []BitRange
   690  	base, size := uint(0), uint(0)
   691  	for i := uint(0); i < uint(len(ba))*64; i++ {
   692  		if ba.get(i) != bb.get(i) {
   693  			if size == 0 {
   694  				base = i
   695  			}
   696  			size++
   697  		} else {
   698  			if size != 0 {
   699  				d = append(d, BitRange{base, size})
   700  			}
   701  			size = 0
   702  		}
   703  	}
   704  	if size != 0 {
   705  		d = append(d, BitRange{base, size})
   706  	}
   707  	return d
   708  }
   709  
   710  // StringifyPallocBits gets the bits in the bit range r from b,
   711  // and returns a string containing the bits as ASCII 0 and 1
   712  // characters.
   713  func StringifyPallocBits(b *PallocBits, r BitRange) string {
   714  	str := ""
   715  	for j := r.I; j < r.I+r.N; j++ {
   716  		if (*pageBits)(b).get(j) != 0 {
   717  			str += "1"
   718  		} else {
   719  			str += "0"
   720  		}
   721  	}
   722  	return str
   723  }
   724  
   725  // Expose pallocData for testing.
   726  type PallocData pallocData
   727  
   728  func (d *PallocData) FindScavengeCandidate(searchIdx uint, min, max uintptr) (uint, uint) {
   729  	return (*pallocData)(d).findScavengeCandidate(searchIdx, min, max)
   730  }
   731  func (d *PallocData) AllocRange(i, n uint) { (*pallocData)(d).allocRange(i, n) }
   732  func (d *PallocData) ScavengedSetRange(i, n uint) {
   733  	(*pallocData)(d).scavenged.setRange(i, n)
   734  }
   735  func (d *PallocData) PallocBits() *PallocBits {
   736  	return (*PallocBits)(&(*pallocData)(d).pallocBits)
   737  }
   738  func (d *PallocData) Scavenged() *PallocBits {
   739  	return (*PallocBits)(&(*pallocData)(d).scavenged)
   740  }
   741  
   742  // Expose fillAligned for testing.
   743  func FillAligned(x uint64, m uint) uint64 { return fillAligned(x, m) }
   744  
   745  // Expose pageCache for testing.
   746  type PageCache pageCache
   747  
   748  const PageCachePages = pageCachePages
   749  
   750  func NewPageCache(base uintptr, cache, scav uint64) PageCache {
   751  	return PageCache(pageCache{base: base, cache: cache, scav: scav})
   752  }
   753  func (c *PageCache) Empty() bool   { return (*pageCache)(c).empty() }
   754  func (c *PageCache) Base() uintptr { return (*pageCache)(c).base }
   755  func (c *PageCache) Cache() uint64 { return (*pageCache)(c).cache }
   756  func (c *PageCache) Scav() uint64  { return (*pageCache)(c).scav }
   757  func (c *PageCache) Alloc(npages uintptr) (uintptr, uintptr) {
   758  	return (*pageCache)(c).alloc(npages)
   759  }
   760  func (c *PageCache) Flush(s *PageAlloc) {
   761  	cp := (*pageCache)(c)
   762  	sp := (*pageAlloc)(s)
   763  
   764  	systemstack(func() {
   765  		// None of the tests need any higher-level locking, so we just
   766  		// take the lock internally.
   767  		lock(sp.mheapLock)
   768  		cp.flush(sp)
   769  		unlock(sp.mheapLock)
   770  	})
   771  }
   772  
   773  // Expose chunk index type.
   774  type ChunkIdx chunkIdx
   775  
   776  // Expose pageAlloc for testing. Note that because pageAlloc is
   777  // not in the heap, so is PageAlloc.
   778  type PageAlloc pageAlloc
   779  
   780  func (p *PageAlloc) Alloc(npages uintptr) (uintptr, uintptr) {
   781  	pp := (*pageAlloc)(p)
   782  
   783  	var addr, scav uintptr
   784  	systemstack(func() {
   785  		// None of the tests need any higher-level locking, so we just
   786  		// take the lock internally.
   787  		lock(pp.mheapLock)
   788  		addr, scav = pp.alloc(npages)
   789  		unlock(pp.mheapLock)
   790  	})
   791  	return addr, scav
   792  }
   793  func (p *PageAlloc) AllocToCache() PageCache {
   794  	pp := (*pageAlloc)(p)
   795  
   796  	var c PageCache
   797  	systemstack(func() {
   798  		// None of the tests need any higher-level locking, so we just
   799  		// take the lock internally.
   800  		lock(pp.mheapLock)
   801  		c = PageCache(pp.allocToCache())
   802  		unlock(pp.mheapLock)
   803  	})
   804  	return c
   805  }
   806  func (p *PageAlloc) Free(base, npages uintptr) {
   807  	pp := (*pageAlloc)(p)
   808  
   809  	systemstack(func() {
   810  		// None of the tests need any higher-level locking, so we just
   811  		// take the lock internally.
   812  		lock(pp.mheapLock)
   813  		pp.free(base, npages)
   814  		unlock(pp.mheapLock)
   815  	})
   816  }
   817  func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) {
   818  	return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end)
   819  }
   820  func (p *PageAlloc) Scavenge(nbytes uintptr, mayUnlock bool) (r uintptr) {
   821  	pp := (*pageAlloc)(p)
   822  	systemstack(func() {
   823  		// None of the tests need any higher-level locking, so we just
   824  		// take the lock internally.
   825  		lock(pp.mheapLock)
   826  		r = pp.scavenge(nbytes, mayUnlock)
   827  		unlock(pp.mheapLock)
   828  	})
   829  	return
   830  }
   831  func (p *PageAlloc) InUse() []AddrRange {
   832  	ranges := make([]AddrRange, 0, len(p.inUse.ranges))
   833  	for _, r := range p.inUse.ranges {
   834  		ranges = append(ranges, AddrRange{r})
   835  	}
   836  	return ranges
   837  }
   838  
   839  // Returns nil if the PallocData's L2 is missing.
   840  func (p *PageAlloc) PallocData(i ChunkIdx) *PallocData {
   841  	ci := chunkIdx(i)
   842  	return (*PallocData)((*pageAlloc)(p).tryChunkOf(ci))
   843  }
   844  
   845  // AddrRange is a wrapper around addrRange for testing.
   846  type AddrRange struct {
   847  	addrRange
   848  }
   849  
   850  // MakeAddrRange creates a new address range.
   851  func MakeAddrRange(base, limit uintptr) AddrRange {
   852  	return AddrRange{makeAddrRange(base, limit)}
   853  }
   854  
   855  // Base returns the virtual base address of the address range.
   856  func (a AddrRange) Base() uintptr {
   857  	return a.addrRange.base.addr()
   858  }
   859  
   860  // Base returns the virtual address of the limit of the address range.
   861  func (a AddrRange) Limit() uintptr {
   862  	return a.addrRange.limit.addr()
   863  }
   864  
   865  // Equals returns true if the two address ranges are exactly equal.
   866  func (a AddrRange) Equals(b AddrRange) bool {
   867  	return a == b
   868  }
   869  
   870  // Size returns the size in bytes of the address range.
   871  func (a AddrRange) Size() uintptr {
   872  	return a.addrRange.size()
   873  }
   874  
   875  // AddrRanges is a wrapper around addrRanges for testing.
   876  type AddrRanges struct {
   877  	addrRanges
   878  	mutable bool
   879  }
   880  
   881  // NewAddrRanges creates a new empty addrRanges.
   882  //
   883  // Note that this initializes addrRanges just like in the
   884  // runtime, so its memory is persistentalloc'd. Call this
   885  // function sparingly since the memory it allocates is
   886  // leaked.
   887  //
   888  // This AddrRanges is mutable, so we can test methods like
   889  // Add.
   890  func NewAddrRanges() AddrRanges {
   891  	r := addrRanges{}
   892  	r.init(new(sysMemStat))
   893  	return AddrRanges{r, true}
   894  }
   895  
   896  // MakeAddrRanges creates a new addrRanges populated with
   897  // the ranges in a.
   898  //
   899  // The returned AddrRanges is immutable, so methods like
   900  // Add will fail.
   901  func MakeAddrRanges(a ...AddrRange) AddrRanges {
   902  	// Methods that manipulate the backing store of addrRanges.ranges should
   903  	// not be used on the result from this function (e.g. add) since they may
   904  	// trigger reallocation. That would normally be fine, except the new
   905  	// backing store won't come from the heap, but from persistentalloc, so
   906  	// we'll leak some memory implicitly.
   907  	ranges := make([]addrRange, 0, len(a))
   908  	total := uintptr(0)
   909  	for _, r := range a {
   910  		ranges = append(ranges, r.addrRange)
   911  		total += r.Size()
   912  	}
   913  	return AddrRanges{addrRanges{
   914  		ranges:     ranges,
   915  		totalBytes: total,
   916  		sysStat:    new(sysMemStat),
   917  	}, false}
   918  }
   919  
   920  // Ranges returns a copy of the ranges described by the
   921  // addrRanges.
   922  func (a *AddrRanges) Ranges() []AddrRange {
   923  	result := make([]AddrRange, 0, len(a.addrRanges.ranges))
   924  	for _, r := range a.addrRanges.ranges {
   925  		result = append(result, AddrRange{r})
   926  	}
   927  	return result
   928  }
   929  
   930  // FindSucc returns the successor to base. See addrRanges.findSucc
   931  // for more details.
   932  func (a *AddrRanges) FindSucc(base uintptr) int {
   933  	return a.findSucc(base)
   934  }
   935  
   936  // Add adds a new AddrRange to the AddrRanges.
   937  //
   938  // The AddrRange must be mutable (i.e. created by NewAddrRanges),
   939  // otherwise this method will throw.
   940  func (a *AddrRanges) Add(r AddrRange) {
   941  	if !a.mutable {
   942  		throw("attempt to mutate immutable AddrRanges")
   943  	}
   944  	a.add(r.addrRange)
   945  }
   946  
   947  // TotalBytes returns the totalBytes field of the addrRanges.
   948  func (a *AddrRanges) TotalBytes() uintptr {
   949  	return a.addrRanges.totalBytes
   950  }
   951  
   952  // BitRange represents a range over a bitmap.
   953  type BitRange struct {
   954  	I, N uint // bit index and length in bits
   955  }
   956  
   957  // NewPageAlloc creates a new page allocator for testing and
   958  // initializes it with the scav and chunks maps. Each key in these maps
   959  // represents a chunk index and each value is a series of bit ranges to
   960  // set within each bitmap's chunk.
   961  //
   962  // The initialization of the pageAlloc preserves the invariant that if a
   963  // scavenged bit is set the alloc bit is necessarily unset, so some
   964  // of the bits described by scav may be cleared in the final bitmap if
   965  // ranges in chunks overlap with them.
   966  //
   967  // scav is optional, and if nil, the scavenged bitmap will be cleared
   968  // (as opposed to all 1s, which it usually is). Furthermore, every
   969  // chunk index in scav must appear in chunks; ones that do not are
   970  // ignored.
   971  func NewPageAlloc(chunks, scav map[ChunkIdx][]BitRange) *PageAlloc {
   972  	p := new(pageAlloc)
   973  
   974  	// We've got an entry, so initialize the pageAlloc.
   975  	p.init(new(mutex), nil)
   976  	lockInit(p.mheapLock, lockRankMheap)
   977  	p.test = true
   978  
   979  	for i, init := range chunks {
   980  		addr := chunkBase(chunkIdx(i))
   981  
   982  		// Mark the chunk's existence in the pageAlloc.
   983  		systemstack(func() {
   984  			lock(p.mheapLock)
   985  			p.grow(addr, pallocChunkBytes)
   986  			unlock(p.mheapLock)
   987  		})
   988  
   989  		// Initialize the bitmap and update pageAlloc metadata.
   990  		chunk := p.chunkOf(chunkIndex(addr))
   991  
   992  		// Clear all the scavenged bits which grow set.
   993  		chunk.scavenged.clearRange(0, pallocChunkPages)
   994  
   995  		// Apply scavenge state if applicable.
   996  		if scav != nil {
   997  			if scvg, ok := scav[i]; ok {
   998  				for _, s := range scvg {
   999  					// Ignore the case of s.N == 0. setRange doesn't handle
  1000  					// it and it's a no-op anyway.
  1001  					if s.N != 0 {
  1002  						chunk.scavenged.setRange(s.I, s.N)
  1003  					}
  1004  				}
  1005  			}
  1006  		}
  1007  
  1008  		// Apply alloc state.
  1009  		for _, s := range init {
  1010  			// Ignore the case of s.N == 0. allocRange doesn't handle
  1011  			// it and it's a no-op anyway.
  1012  			if s.N != 0 {
  1013  				chunk.allocRange(s.I, s.N)
  1014  			}
  1015  		}
  1016  
  1017  		// Update heap metadata for the allocRange calls above.
  1018  		systemstack(func() {
  1019  			lock(p.mheapLock)
  1020  			p.update(addr, pallocChunkPages, false, false)
  1021  			unlock(p.mheapLock)
  1022  		})
  1023  	}
  1024  
  1025  	systemstack(func() {
  1026  		lock(p.mheapLock)
  1027  		p.scavengeStartGen()
  1028  		unlock(p.mheapLock)
  1029  	})
  1030  
  1031  	return (*PageAlloc)(p)
  1032  }
  1033  
  1034  // FreePageAlloc releases hard OS resources owned by the pageAlloc. Once this
  1035  // is called the pageAlloc may no longer be used. The object itself will be
  1036  // collected by the garbage collector once it is no longer live.
  1037  func FreePageAlloc(pp *PageAlloc) {
  1038  	p := (*pageAlloc)(pp)
  1039  
  1040  	// Free all the mapped space for the summary levels.
  1041  	if pageAlloc64Bit != 0 {
  1042  		for l := 0; l < summaryLevels; l++ {
  1043  			sysFree(unsafe.Pointer(&p.summary[l][0]), uintptr(cap(p.summary[l]))*pallocSumBytes, nil)
  1044  		}
  1045  	} else {
  1046  		resSize := uintptr(0)
  1047  		for _, s := range p.summary {
  1048  			resSize += uintptr(cap(s)) * pallocSumBytes
  1049  		}
  1050  		sysFree(unsafe.Pointer(&p.summary[0][0]), alignUp(resSize, physPageSize), nil)
  1051  	}
  1052  
  1053  	// Free the mapped space for chunks.
  1054  	for i := range p.chunks {
  1055  		if x := p.chunks[i]; x != nil {
  1056  			p.chunks[i] = nil
  1057  			// This memory comes from sysAlloc and will always be page-aligned.
  1058  			sysFree(unsafe.Pointer(x), unsafe.Sizeof(*p.chunks[0]), nil)
  1059  		}
  1060  	}
  1061  }
  1062  
  1063  // BaseChunkIdx is a convenient chunkIdx value which works on both
  1064  // 64 bit and 32 bit platforms, allowing the tests to share code
  1065  // between the two.
  1066  //
  1067  // This should not be higher than 0x100*pallocChunkBytes to support
  1068  // mips and mipsle, which only have 31-bit address spaces.
  1069  var BaseChunkIdx = ChunkIdx(chunkIndex(((0xc000*pageAlloc64Bit + 0x100*pageAlloc32Bit) * pallocChunkBytes) + arenaBaseOffset*sys.GoosAix))
  1070  
  1071  // PageBase returns an address given a chunk index and a page index
  1072  // relative to that chunk.
  1073  func PageBase(c ChunkIdx, pageIdx uint) uintptr {
  1074  	return chunkBase(chunkIdx(c)) + uintptr(pageIdx)*pageSize
  1075  }
  1076  
  1077  type BitsMismatch struct {
  1078  	Base      uintptr
  1079  	Got, Want uint64
  1080  }
  1081  
  1082  func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) {
  1083  	ok = true
  1084  
  1085  	// Run on the system stack to avoid stack growth allocation.
  1086  	systemstack(func() {
  1087  		getg().m.mallocing++
  1088  
  1089  		// Lock so that we can safely access the bitmap.
  1090  		lock(&mheap_.lock)
  1091  	chunkLoop:
  1092  		for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
  1093  			chunk := mheap_.pages.tryChunkOf(i)
  1094  			if chunk == nil {
  1095  				continue
  1096  			}
  1097  			for j := 0; j < pallocChunkPages/64; j++ {
  1098  				// Run over each 64-bit bitmap section and ensure
  1099  				// scavenged is being cleared properly on allocation.
  1100  				// If a used bit and scavenged bit are both set, that's
  1101  				// an error, and could indicate a larger problem, or
  1102  				// an accounting problem.
  1103  				want := chunk.scavenged[j] &^ chunk.pallocBits[j]
  1104  				got := chunk.scavenged[j]
  1105  				if want != got {
  1106  					ok = false
  1107  					if n >= len(mismatches) {
  1108  						break chunkLoop
  1109  					}
  1110  					mismatches[n] = BitsMismatch{
  1111  						Base: chunkBase(i) + uintptr(j)*64*pageSize,
  1112  						Got:  got,
  1113  						Want: want,
  1114  					}
  1115  					n++
  1116  				}
  1117  			}
  1118  		}
  1119  		unlock(&mheap_.lock)
  1120  
  1121  		getg().m.mallocing--
  1122  	})
  1123  	return
  1124  }
  1125  
  1126  func PageCachePagesLeaked() (leaked uintptr) {
  1127  	stopTheWorld("PageCachePagesLeaked")
  1128  
  1129  	// Walk over destroyed Ps and look for unflushed caches.
  1130  	deadp := allp[len(allp):cap(allp)]
  1131  	for _, p := range deadp {
  1132  		// Since we're going past len(allp) we may see nil Ps.
  1133  		// Just ignore them.
  1134  		if p != nil {
  1135  			leaked += uintptr(sys.OnesCount64(p.pcache.cache))
  1136  		}
  1137  	}
  1138  
  1139  	startTheWorld()
  1140  	return
  1141  }
  1142  
  1143  var Semacquire = semacquire
  1144  var Semrelease1 = semrelease1
  1145  
  1146  func SemNwait(addr *uint32) uint32 {
  1147  	root := semroot(addr)
  1148  	return atomic.Load(&root.nwait)
  1149  }
  1150  
  1151  // mspan wrapper for testing.
  1152  //go:notinheap
  1153  type MSpan mspan
  1154  
  1155  // Allocate an mspan for testing.
  1156  func AllocMSpan() *MSpan {
  1157  	var s *mspan
  1158  	systemstack(func() {
  1159  		lock(&mheap_.lock)
  1160  		s = (*mspan)(mheap_.spanalloc.alloc())
  1161  		unlock(&mheap_.lock)
  1162  	})
  1163  	return (*MSpan)(s)
  1164  }
  1165  
  1166  // Free an allocated mspan.
  1167  func FreeMSpan(s *MSpan) {
  1168  	systemstack(func() {
  1169  		lock(&mheap_.lock)
  1170  		mheap_.spanalloc.free(unsafe.Pointer(s))
  1171  		unlock(&mheap_.lock)
  1172  	})
  1173  }
  1174  
  1175  func MSpanCountAlloc(ms *MSpan, bits []byte) int {
  1176  	s := (*mspan)(ms)
  1177  	s.nelems = uintptr(len(bits) * 8)
  1178  	s.gcmarkBits = (*gcBits)(unsafe.Pointer(&bits[0]))
  1179  	result := s.countAlloc()
  1180  	s.gcmarkBits = nil
  1181  	return result
  1182  }
  1183  
  1184  const (
  1185  	TimeHistSubBucketBits   = timeHistSubBucketBits
  1186  	TimeHistNumSubBuckets   = timeHistNumSubBuckets
  1187  	TimeHistNumSuperBuckets = timeHistNumSuperBuckets
  1188  )
  1189  
  1190  type TimeHistogram timeHistogram
  1191  
  1192  // Counts returns the counts for the given bucket, subBucket indices.
  1193  // Returns true if the bucket was valid, otherwise returns the counts
  1194  // for the underflow bucket and false.
  1195  func (th *TimeHistogram) Count(bucket, subBucket uint) (uint64, bool) {
  1196  	t := (*timeHistogram)(th)
  1197  	i := bucket*TimeHistNumSubBuckets + subBucket
  1198  	if i >= uint(len(t.counts)) {
  1199  		return t.underflow, false
  1200  	}
  1201  	return t.counts[i], true
  1202  }
  1203  
  1204  func (th *TimeHistogram) Record(duration int64) {
  1205  	(*timeHistogram)(th).record(duration)
  1206  }
  1207  
  1208  func SetIntArgRegs(a int) int {
  1209  	lock(&finlock)
  1210  	old := intArgRegs
  1211  	if a >= 0 {
  1212  		intArgRegs = a
  1213  	}
  1214  	unlock(&finlock)
  1215  	return old
  1216  }
  1217  
  1218  func FinalizerGAsleep() bool {
  1219  	lock(&finlock)
  1220  	result := fingwait
  1221  	unlock(&finlock)
  1222  	return result
  1223  }
  1224  
  1225  // For GCTestMoveStackOnNextCall, it's important not to introduce an
  1226  // extra layer of call, since then there's a return before the "real"
  1227  // next call.
  1228  var GCTestMoveStackOnNextCall = gcTestMoveStackOnNextCall
  1229  
  1230  // For GCTestIsReachable, it's important that we do this as a call so
  1231  // escape analysis can see through it.
  1232  func GCTestIsReachable(ptrs ...unsafe.Pointer) (mask uint64) {
  1233  	return gcTestIsReachable(ptrs...)
  1234  }
  1235  
  1236  // For GCTestPointerClass, it's important that we do this as a call so
  1237  // escape analysis can see through it.
  1238  //
  1239  // This is nosplit because gcTestPointerClass is.
  1240  //
  1241  //go:nosplit
  1242  func GCTestPointerClass(p unsafe.Pointer) string {
  1243  	return gcTestPointerClass(p)
  1244  }
  1245  
  1246  const Raceenabled = raceenabled
  1247  

View as plain text