race1.go 8.31 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Implementation of the race detector API.
// +build race

package runtime

import "unsafe"

// Race runtime functions called via runtime·racecall.
//go:linkname __tsan_init __tsan_init
var __tsan_init byte

//go:linkname __tsan_fini __tsan_fini
var __tsan_fini byte

//go:linkname __tsan_map_shadow __tsan_map_shadow
var __tsan_map_shadow byte

//go:linkname __tsan_finalizer_goroutine __tsan_finalizer_goroutine
var __tsan_finalizer_goroutine byte

//go:linkname __tsan_go_start __tsan_go_start
var __tsan_go_start byte

//go:linkname __tsan_go_end __tsan_go_end
var __tsan_go_end byte

//go:linkname __tsan_malloc __tsan_malloc
var __tsan_malloc byte

//go:linkname __tsan_acquire __tsan_acquire
var __tsan_acquire byte

//go:linkname __tsan_release __tsan_release
var __tsan_release byte

//go:linkname __tsan_release_merge __tsan_release_merge
var __tsan_release_merge byte

//go:linkname __tsan_go_ignore_sync_begin __tsan_go_ignore_sync_begin
var __tsan_go_ignore_sync_begin byte

//go:linkname __tsan_go_ignore_sync_end __tsan_go_ignore_sync_end
var __tsan_go_ignore_sync_end byte

// Mimic what cmd/cgo would do.
//go:cgo_import_static __tsan_init
//go:cgo_import_static __tsan_fini
//go:cgo_import_static __tsan_map_shadow
//go:cgo_import_static __tsan_finalizer_goroutine
//go:cgo_import_static __tsan_go_start
//go:cgo_import_static __tsan_go_end
//go:cgo_import_static __tsan_malloc
//go:cgo_import_static __tsan_acquire
//go:cgo_import_static __tsan_release
//go:cgo_import_static __tsan_release_merge
//go:cgo_import_static __tsan_go_ignore_sync_begin
//go:cgo_import_static __tsan_go_ignore_sync_end

// These are called from race_amd64.s.
//go:cgo_import_static __tsan_read
//go:cgo_import_static __tsan_read_pc
//go:cgo_import_static __tsan_read_range
//go:cgo_import_static __tsan_write
//go:cgo_import_static __tsan_write_pc
//go:cgo_import_static __tsan_write_range
//go:cgo_import_static __tsan_func_enter
//go:cgo_import_static __tsan_func_exit

//go:cgo_import_static __tsan_go_atomic32_load
//go:cgo_import_static __tsan_go_atomic64_load
//go:cgo_import_static __tsan_go_atomic32_store
//go:cgo_import_static __tsan_go_atomic64_store
//go:cgo_import_static __tsan_go_atomic32_exchange
//go:cgo_import_static __tsan_go_atomic64_exchange
//go:cgo_import_static __tsan_go_atomic32_fetch_add
//go:cgo_import_static __tsan_go_atomic64_fetch_add
//go:cgo_import_static __tsan_go_atomic32_compare_exchange
//go:cgo_import_static __tsan_go_atomic64_compare_exchange

84 85 86 87
// start/end of global data (data+bss).
var racedatastart uintptr
var racedataend uintptr

88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
// start/end of heap for race_amd64.s
var racearenastart uintptr
var racearenaend uintptr

func racefuncenter(uintptr)
func racefuncexit()
func racereadrangepc1(uintptr, uintptr, uintptr)
func racewriterangepc1(uintptr, uintptr, uintptr)
func racesymbolizethunk(uintptr)

// racecall allows calling an arbitrary function f from C race runtime
// with up to 4 uintptr arguments.
func racecall(*byte, uintptr, uintptr, uintptr, uintptr)

// checks if the address has shadow (i.e. heap or data/bss)
//go:nosplit
func isvalidaddr(addr unsafe.Pointer) bool {
	return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend ||
106
		racedatastart <= uintptr(addr) && uintptr(addr) < racedataend
107 108 109 110 111 112
}

//go:nosplit
func raceinit() uintptr {
	// cgo is required to initialize libc, which is used by race runtime
	if !iscgo {
113
		throw("raceinit: race build must use cgo")
114 115 116 117 118 119
	}

	var racectx uintptr
	racecall(&__tsan_init, uintptr(unsafe.Pointer(&racectx)), funcPC(racesymbolizethunk), 0, 0)

	// Round data segment to page boundaries, because it's used in mmap().
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
	start := ^uintptr(0)
	end := uintptr(0)
	if start > uintptr(unsafe.Pointer(&noptrdata)) {
		start = uintptr(unsafe.Pointer(&noptrdata))
	}
	if start > uintptr(unsafe.Pointer(&data)) {
		start = uintptr(unsafe.Pointer(&data))
	}
	if start > uintptr(unsafe.Pointer(&noptrbss)) {
		start = uintptr(unsafe.Pointer(&noptrbss))
	}
	if start > uintptr(unsafe.Pointer(&bss)) {
		start = uintptr(unsafe.Pointer(&bss))
	}
	if end < uintptr(unsafe.Pointer(&enoptrdata)) {
		end = uintptr(unsafe.Pointer(&enoptrdata))
	}
	if end < uintptr(unsafe.Pointer(&edata)) {
		end = uintptr(unsafe.Pointer(&edata))
	}
	if end < uintptr(unsafe.Pointer(&enoptrbss)) {
		end = uintptr(unsafe.Pointer(&enoptrbss))
	}
	if end < uintptr(unsafe.Pointer(&ebss)) {
		end = uintptr(unsafe.Pointer(&ebss))
	}
	size := round(end-start, _PageSize)
147
	racecall(&__tsan_map_shadow, start, size, 0, 0)
148 149
	racedatastart = start
	racedataend = start + size
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335

	return racectx
}

//go:nosplit
func racefini() {
	racecall(&__tsan_fini, 0, 0, 0, 0)
}

//go:nosplit
func racemapshadow(addr unsafe.Pointer, size uintptr) {
	if racearenastart == 0 {
		racearenastart = uintptr(addr)
	}
	if racearenaend < uintptr(addr)+size {
		racearenaend = uintptr(addr) + size
	}
	racecall(&__tsan_map_shadow, uintptr(addr), size, 0, 0)
}

//go:nosplit
func racemalloc(p unsafe.Pointer, sz uintptr) {
	racecall(&__tsan_malloc, uintptr(p), sz, 0, 0)
}

//go:nosplit
func racegostart(pc uintptr) uintptr {
	_g_ := getg()
	var spawng *g
	if _g_.m.curg != nil {
		spawng = _g_.m.curg
	} else {
		spawng = _g_
	}

	var racectx uintptr
	racecall(&__tsan_go_start, spawng.racectx, uintptr(unsafe.Pointer(&racectx)), pc, 0)
	return racectx
}

//go:nosplit
func racegoend() {
	racecall(&__tsan_go_end, getg().racectx, 0, 0, 0)
}

//go:nosplit
func racewriterangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
	_g_ := getg()
	if _g_ != _g_.m.curg {
		// The call is coming from manual instrumentation of Go code running on g0/gsignal.
		// Not interesting.
		return
	}
	if callpc != 0 {
		racefuncenter(callpc)
	}
	racewriterangepc1(uintptr(addr), sz, pc)
	if callpc != 0 {
		racefuncexit()
	}
}

//go:nosplit
func racereadrangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
	_g_ := getg()
	if _g_ != _g_.m.curg {
		// The call is coming from manual instrumentation of Go code running on g0/gsignal.
		// Not interesting.
		return
	}
	if callpc != 0 {
		racefuncenter(callpc)
	}
	racereadrangepc1(uintptr(addr), sz, pc)
	if callpc != 0 {
		racefuncexit()
	}
}

//go:nosplit
func racewriteobjectpc(addr unsafe.Pointer, t *_type, callpc, pc uintptr) {
	kind := t.kind & _KindMask
	if kind == _KindArray || kind == _KindStruct {
		racewriterangepc(addr, t.size, callpc, pc)
	} else {
		racewritepc(addr, callpc, pc)
	}
}

//go:nosplit
func racereadobjectpc(addr unsafe.Pointer, t *_type, callpc, pc uintptr) {
	kind := t.kind & _KindMask
	if kind == _KindArray || kind == _KindStruct {
		racereadrangepc(addr, t.size, callpc, pc)
	} else {
		racereadpc(addr, callpc, pc)
	}
}

//go:nosplit
func raceacquire(addr unsafe.Pointer) {
	raceacquireg(getg(), addr)
}

//go:nosplit
func raceacquireg(gp *g, addr unsafe.Pointer) {
	if getg().raceignore != 0 || !isvalidaddr(addr) {
		return
	}
	racecall(&__tsan_acquire, gp.racectx, uintptr(addr), 0, 0)
}

//go:nosplit
func racerelease(addr unsafe.Pointer) {
	_g_ := getg()
	if _g_.raceignore != 0 || !isvalidaddr(addr) {
		return
	}
	racereleaseg(_g_, addr)
}

//go:nosplit
func racereleaseg(gp *g, addr unsafe.Pointer) {
	if getg().raceignore != 0 || !isvalidaddr(addr) {
		return
	}
	racecall(&__tsan_release, gp.racectx, uintptr(addr), 0, 0)
}

//go:nosplit
func racereleasemerge(addr unsafe.Pointer) {
	racereleasemergeg(getg(), addr)
}

//go:nosplit
func racereleasemergeg(gp *g, addr unsafe.Pointer) {
	if getg().raceignore != 0 || !isvalidaddr(addr) {
		return
	}
	racecall(&__tsan_release_merge, gp.racectx, uintptr(addr), 0, 0)
}

//go:nosplit
func racefingo() {
	racecall(&__tsan_finalizer_goroutine, getg().racectx, 0, 0, 0)
}

//go:nosplit

func RaceAcquire(addr unsafe.Pointer) {
	raceacquire(addr)
}

//go:nosplit

func RaceRelease(addr unsafe.Pointer) {
	racerelease(addr)
}

//go:nosplit

func RaceReleaseMerge(addr unsafe.Pointer) {
	racereleasemerge(addr)
}

//go:nosplit

// RaceEnable re-enables handling of race events in the current goroutine.
func RaceDisable() {
	_g_ := getg()
	if _g_.raceignore == 0 {
		racecall(&__tsan_go_ignore_sync_begin, _g_.racectx, 0, 0, 0)
	}
	_g_.raceignore++
}

//go:nosplit

// RaceDisable disables handling of race events in the current goroutine.
func RaceEnable() {
	_g_ := getg()
	_g_.raceignore--
	if _g_.raceignore == 0 {
		racecall(&__tsan_go_ignore_sync_end, _g_.racectx, 0, 0, 0)
	}
}