Commit 7d14401b authored by unknown's avatar unknown Committed by Michael Matloob

runtime/pprof: write profiles in protobuf format.

Added functions with suffix proto and stuff from pprof tool to translate
to protobuf. Done as the profile proto is more extensible than the legacy
pprof format and is pprof's preferred profile format. Large part was taken
from https://github.com/google/pprof tool. Tested by hand and compared the
result with translated by pprof tool, profiles are identical.
Fixes #16093
Change-Id: I5acdb2809cab0d16ed4694fdaa7b8ddfd68df11e
Reviewed-on: https://go-review.googlesource.com/30556
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarMichael Matloob <matloob@golang.org>
parent d70b0fe6
...@@ -170,12 +170,14 @@ var pkgDeps = map[string][]string{ ...@@ -170,12 +170,14 @@ var pkgDeps = map[string][]string{
"log": {"L1", "os", "fmt", "time"}, "log": {"L1", "os", "fmt", "time"},
// Packages used by testing must be low-level (L2+fmt). // Packages used by testing must be low-level (L2+fmt).
"regexp": {"L2", "regexp/syntax"}, "regexp": {"L2", "regexp/syntax"},
"regexp/syntax": {"L2"}, "regexp/syntax": {"L2"},
"runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"}, "runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"},
"runtime/pprof": {"L2", "fmt", "os", "text/tabwriter"}, "runtime/pprof/internal/profile": {"L2"},
"runtime/trace": {"L0"}, "runtime/pprof/internal/protopprof": {"L2", "fmt", "runtime/pprof/internal/profile", "os", "time"},
"text/tabwriter": {"L2"}, "runtime/pprof": {"L2", "fmt", "runtime/pprof/internal/profile", "runtime/pprof/internal/protopprof", "time"},
"runtime/trace": {"L0"},
"text/tabwriter": {"L2"},
"testing": {"L2", "flag", "fmt", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"}, "testing": {"L2", "flag", "fmt", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"},
"testing/iotest": {"L2", "log"}, "testing/iotest": {"L2", "log"},
......
...@@ -271,10 +271,11 @@ func testCgoPprof(t *testing.T, buildArg, runArg string) { ...@@ -271,10 +271,11 @@ func testCgoPprof(t *testing.T, buildArg, runArg string) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
fn := strings.TrimSpace(string(got)) fn := strings.TrimSpace(string(got))
defer os.Remove(fn) defer os.Remove(fn)
cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", exe, fn)) cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", "-symbolize=force", exe, fn))
found := false found := false
for i, e := range cmd.Env { for i, e := range cmd.Env {
......
This diff is collapsed.
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package profile
import (
"sort"
)
// preEncode populates the unexported fields to be used by encode
// (with suffix X) from the corresponding exported fields. The
// exported fields are cleared up to facilitate testing.
func (p *Profile) preEncode() {
strings := make(map[string]int)
addString(strings, "")
for _, st := range p.SampleType {
st.typeX = addString(strings, st.Type)
st.unitX = addString(strings, st.Unit)
}
for _, s := range p.Sample {
s.labelX = nil
var keys []string
for k := range s.Label {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
vs := s.Label[k]
for _, v := range vs {
s.labelX = append(s.labelX,
Label{
keyX: addString(strings, k),
strX: addString(strings, v),
},
)
}
}
var numKeys []string
for k := range s.NumLabel {
numKeys = append(numKeys, k)
}
sort.Strings(numKeys)
for _, k := range numKeys {
vs := s.NumLabel[k]
for _, v := range vs {
s.labelX = append(s.labelX,
Label{
keyX: addString(strings, k),
numX: v,
},
)
}
}
s.locationIDX = nil
for _, l := range s.Location {
s.locationIDX = append(s.locationIDX, l.ID)
}
}
for _, m := range p.Mapping {
m.fileX = addString(strings, m.File)
m.buildIDX = addString(strings, m.BuildID)
}
for _, l := range p.Location {
for i, ln := range l.Line {
if ln.Function != nil {
l.Line[i].functionIDX = ln.Function.ID
} else {
l.Line[i].functionIDX = 0
}
}
if l.Mapping != nil {
l.mappingIDX = l.Mapping.ID
} else {
l.mappingIDX = 0
}
}
for _, f := range p.Function {
f.nameX = addString(strings, f.Name)
f.systemNameX = addString(strings, f.SystemName)
f.filenameX = addString(strings, f.Filename)
}
if pt := p.PeriodType; pt != nil {
pt.typeX = addString(strings, pt.Type)
pt.unitX = addString(strings, pt.Unit)
}
p.stringTable = make([]string, len(strings))
for s, i := range strings {
p.stringTable[i] = s
}
}
func (p *Profile) encode(b *buffer) {
for _, x := range p.SampleType {
encodeMessage(b, 1, x)
}
for _, x := range p.Sample {
encodeMessage(b, 2, x)
}
for _, x := range p.Mapping {
encodeMessage(b, 3, x)
}
for _, x := range p.Location {
encodeMessage(b, 4, x)
}
for _, x := range p.Function {
encodeMessage(b, 5, x)
}
encodeStrings(b, 6, p.stringTable)
encodeInt64Opt(b, 9, p.TimeNanos)
encodeInt64Opt(b, 10, p.DurationNanos)
if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) {
encodeMessage(b, 11, p.PeriodType)
}
encodeInt64Opt(b, 12, p.Period)
}
func (p *ValueType) encode(b *buffer) {
encodeInt64Opt(b, 1, p.typeX)
encodeInt64Opt(b, 2, p.unitX)
}
func (p *Sample) encode(b *buffer) {
encodeUint64s(b, 1, p.locationIDX)
for _, x := range p.Value {
encodeInt64(b, 2, x)
}
for _, x := range p.labelX {
encodeMessage(b, 3, x)
}
}
func (p Label) encode(b *buffer) {
encodeInt64Opt(b, 1, p.keyX)
encodeInt64Opt(b, 2, p.strX)
encodeInt64Opt(b, 3, p.numX)
}
func (p *Mapping) encode(b *buffer) {
encodeUint64Opt(b, 1, p.ID)
encodeUint64Opt(b, 2, p.Start)
encodeUint64Opt(b, 3, p.Limit)
encodeUint64Opt(b, 4, p.Offset)
encodeInt64Opt(b, 5, p.fileX)
encodeInt64Opt(b, 6, p.buildIDX)
encodeBoolOpt(b, 7, p.HasFunctions)
encodeBoolOpt(b, 8, p.HasFilenames)
encodeBoolOpt(b, 9, p.HasLineNumbers)
encodeBoolOpt(b, 10, p.HasInlineFrames)
}
func (p *Location) encode(b *buffer) {
encodeUint64Opt(b, 1, p.ID)
encodeUint64Opt(b, 2, p.mappingIDX)
encodeUint64Opt(b, 3, p.Address)
for i := range p.Line {
encodeMessage(b, 4, &p.Line[i])
}
}
func (p *Line) encode(b *buffer) {
encodeUint64Opt(b, 1, p.functionIDX)
encodeInt64Opt(b, 2, p.Line)
}
func (p *Function) encode(b *buffer) {
encodeUint64Opt(b, 1, p.ID)
encodeInt64Opt(b, 2, p.nameX)
encodeInt64Opt(b, 3, p.systemNameX)
encodeInt64Opt(b, 4, p.filenameX)
encodeInt64Opt(b, 5, p.StartLine)
}
func addString(strings map[string]int, s string) int64 {
i, ok := strings[s]
if !ok {
i = len(strings)
strings[s] = i
}
return int64(i)
}
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Package profile provides a representation of profile.proto and
// methods to encode/decode profiles in this format.
package profile
import (
"io"
)
// Profile is an in-memory representation of profile.proto.
type Profile struct {
SampleType []*ValueType
Sample []*Sample
Mapping []*Mapping
Location []*Location
Function []*Function
TimeNanos int64
DurationNanos int64
PeriodType *ValueType
Period int64
stringTable []string
}
// ValueType corresponds to Profile.ValueType
type ValueType struct {
Type string // cpu, wall, inuse_space, etc
Unit string // seconds, nanoseconds, bytes, etc
typeX int64
unitX int64
}
// Sample corresponds to Profile.Sample
type Sample struct {
Location []*Location
Value []int64
Label map[string][]string
NumLabel map[string][]int64
locationIDX []uint64
labelX []Label
}
// Label corresponds to Profile.Label
type Label struct {
keyX int64
// Exactly one of the two following values must be set
strX int64
numX int64 // Integer value for this label
}
// Mapping corresponds to Profile.Mapping
type Mapping struct {
ID uint64
Start uint64
Limit uint64
Offset uint64
File string
BuildID string
HasFunctions bool
HasFilenames bool
HasLineNumbers bool
HasInlineFrames bool
fileX int64
buildIDX int64
}
// Location corresponds to Profile.Location
type Location struct {
ID uint64
Mapping *Mapping
Address uint64
Line []Line
mappingIDX uint64
}
// Line corresponds to Profile.Line
type Line struct {
Function *Function
Line int64
functionIDX uint64
}
// Function corresponds to Profile.Function
type Function struct {
ID uint64
Name string
SystemName string
Filename string
StartLine int64
nameX int64
systemNameX int64
filenameX int64
}
// Write writes the profile as a gzip-compressed marshaled protobuf.
func (p *Profile) Write(w io.Writer) error {
p.preEncode()
var b buffer
p.encode(&b)
_, err := w.Write(b.data)
return err
}
package profile
import (
"bufio"
"errors"
"io"
"strconv"
"strings"
)
var errUnrecognized = errors.New("unrecognized profile format")
func hasLibFile(file string) string {
ix := strings.Index(file, "so")
if ix < 1 {
return ""
}
start := ix - 1
end := ix + 2
s := file[start:end]
if end < len(file) {
endalt := end
if file[endalt] != '.' && file[endalt] != '_' {
return s
}
endalt++
for file[endalt] >= '0' && file[endalt] <= '9' {
endalt++
}
if endalt < end+2 {
return s
}
return s[start:endalt]
}
return s
}
// massageMappings applies heuristic-based changes to the profile
// mappings to account for quirks of some environments.
func (p *Profile) massageMappings() {
// Merge adjacent regions with matching names, checking that the offsets match
if len(p.Mapping) > 1 {
mappings := []*Mapping{p.Mapping[0]}
for _, m := range p.Mapping[1:] {
lm := mappings[len(mappings)-1]
if offset := lm.Offset + (lm.Limit - lm.Start); lm.Limit == m.Start &&
offset == m.Offset &&
(lm.File == m.File || lm.File == "") {
lm.File = m.File
lm.Limit = m.Limit
if lm.BuildID == "" {
lm.BuildID = m.BuildID
}
p.updateLocationMapping(m, lm)
continue
}
mappings = append(mappings, m)
}
p.Mapping = mappings
}
// Use heuristics to identify main binary and move it to the top of the list of mappings
for i, m := range p.Mapping {
file := strings.TrimSpace(strings.Replace(m.File, "(deleted)", "", -1))
if len(file) == 0 {
continue
}
if len(hasLibFile(file)) > 0 {
continue
}
if strings.HasPrefix(file, "[") {
continue
}
// Swap what we guess is main to position 0.
p.Mapping[0], p.Mapping[i] = p.Mapping[i], p.Mapping[0]
break
}
// Keep the mapping IDs neatly sorted
for i, m := range p.Mapping {
m.ID = uint64(i + 1)
}
}
func (p *Profile) updateLocationMapping(from, to *Mapping) {
for _, l := range p.Location {
if l.Mapping == from {
l.Mapping = to
}
}
}
// remapLocationIDs ensures there is a location for each address
// referenced by a sample, and remaps the samples to point to the new
// location ids.
func (p *Profile) remapLocationIDs() {
seen := make(map[*Location]bool, len(p.Location))
var locs []*Location
for _, s := range p.Sample {
for _, l := range s.Location {
if seen[l] {
continue
}
l.ID = uint64(len(locs) + 1)
locs = append(locs, l)
seen[l] = true
}
}
p.Location = locs
}
func (p *Profile) remapFunctionIDs() {
seen := make(map[*Function]bool, len(p.Function))
var fns []*Function
for _, l := range p.Location {
for _, ln := range l.Line {
fn := ln.Function
if fn == nil || seen[fn] {
continue
}
fn.ID = uint64(len(fns) + 1)
fns = append(fns, fn)
seen[fn] = true
}
}
p.Function = fns
}
// remapMappingIDs matches location addresses with existing mappings
// and updates them appropriately. This is O(N*M), if this ever shows
// up as a bottleneck, evaluate sorting the mappings and doing a
// binary search, which would make it O(N*log(M)).
func (p *Profile) remapMappingIDs() {
// Some profile handlers will incorrectly set regions for the main
// executable if its section is remapped. Fix them through heuristics.
if len(p.Mapping) > 0 {
// Remove the initial mapping if named '/anon_hugepage' and has a
// consecutive adjacent mapping.
if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") {
if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start {
p.Mapping = p.Mapping[1:]
}
}
}
// Subtract the offset from the start of the main mapping if it
// ends up at a recognizable start address.
if len(p.Mapping) > 0 {
const expectedStart = 0x400000
if m := p.Mapping[0]; m.Start-m.Offset == expectedStart {
m.Start = expectedStart
m.Offset = 0
}
}
// Associate each location with an address to the corresponding
// mapping. Create fake mapping if a suitable one isn't found.
var fake *Mapping
nextLocation:
for _, l := range p.Location {
a := l.Address
if l.Mapping != nil || a == 0 {
continue
}
for _, m := range p.Mapping {
if m.Start <= a && a < m.Limit {
l.Mapping = m
continue nextLocation
}
}
// Work around legacy handlers failing to encode the first
// part of mappings split into adjacent ranges.
for _, m := range p.Mapping {
if m.Offset != 0 && m.Start-m.Offset <= a && a < m.Start {
m.Start -= m.Offset
m.Offset = 0
l.Mapping = m
continue nextLocation
}
}
// If there is still no mapping, create a fake one.
// This is important for the Go legacy handler, which produced
// no mappings.
if fake == nil {
fake = &Mapping{
ID: 1,
Limit: ^uint64(0),
}
p.Mapping = append(p.Mapping, fake)
}
l.Mapping = fake
}
// Reset all mapping IDs.
for i, m := range p.Mapping {
m.ID = uint64(i + 1)
}
}
func (p *Profile) RemapAll() {
p.remapLocationIDs()
p.remapFunctionIDs()
p.remapMappingIDs()
}
// ParseProcMaps parses a memory map in the format of /proc/self/maps.
// ParseMemoryMap should be called after setting on a profile to
// associate locations to the corresponding mapping based on their
// address.
func ParseProcMaps(rd io.Reader) ([]*Mapping, error) {
var mapping []*Mapping
b := bufio.NewReader(rd)
var attrs []string
var r *strings.Replacer
const delimiter = "="
for {
l, err := b.ReadString('\n')
if err != nil {
if err != io.EOF {
return nil, err
}
if l == "" {
break
}
}
if l = strings.TrimSpace(l); l == "" {
continue
}
if r != nil {
l = r.Replace(l)
}
m, err := parseMappingEntry(l)
if err != nil {
if err == errUnrecognized {
// Recognize assignments of the form: attr=value, and replace
// $attr with value on subsequent mappings.
if attr := strings.SplitN(l, delimiter, 2); len(attr) == 2 {
attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]))
r = strings.NewReplacer(attrs...)
}
// Ignore any unrecognized entries
continue
}
return nil, err
}
if m == nil {
continue
}
mapping = append(mapping, m)
}
return mapping, nil
}
// ParseMemoryMap parses a memory map in the format of
// /proc/self/maps, and overrides the mappings in the current profile.
// It renumbers the samples and locations in the profile correspondingly.
func (p *Profile) ParseMemoryMap(rd io.Reader) error {
mapping, err := ParseProcMaps(rd)
if err != nil {
return err
}
p.Mapping = append(p.Mapping, mapping...)
p.massageMappings()
p.RemapAll()
return nil
}
func parseMappingEntry(l string) (*Mapping, error) {
mapping := &Mapping{}
var err error
fields := strings.Fields(l)
// fmt.Println(len(me), me)
if len(fields) == 6 {
if !strings.Contains(fields[1], "x") {
// Skip non-executable entries.
return nil, nil
}
addrRange := strings.Split(fields[0], "-")
if mapping.Start, err = strconv.ParseUint(addrRange[0], 16, 64); err != nil {
return nil, errUnrecognized
}
if mapping.Limit, err = strconv.ParseUint(addrRange[1], 16, 64); err != nil {
return nil, errUnrecognized
}
offset := fields[2]
if offset != "" {
if mapping.Offset, err = strconv.ParseUint(offset, 16, 64); err != nil {
return nil, errUnrecognized
}
}
mapping.File = fields[5]
return mapping, nil
}
return nil, errUnrecognized
}
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// This file is a simple protocol buffer encoder and decoder.
//
// A protocol message must implement the message interface:
// decoder() []decoder
// encode(*buffer)
//
// The decode method returns a slice indexed by field number that gives the
// function to decode that field.
// The encode method encodes its receiver into the given buffer.
//
// The two methods are simple enough to be implemented by hand rather than
// by using a protocol compiler.
//
// See profile.go for examples of messages implementing this interface.
//
// There is no support for groups, message sets, or "has" bits.
package profile
type buffer struct {
field int
typ int
u64 uint64
data []byte
tmp [16]byte
}
type message interface {
encode(*buffer)
}
func encodeVarint(b *buffer, x uint64) {
for x >= 128 {
b.data = append(b.data, byte(x)|0x80)
x >>= 7
}
b.data = append(b.data, byte(x))
}
func encodeLength(b *buffer, tag int, len int) {
encodeVarint(b, uint64(tag)<<3|2)
encodeVarint(b, uint64(len))
}
func encodeUint64(b *buffer, tag int, x uint64) {
// append varint to b.data
encodeVarint(b, uint64(tag)<<3|0)
encodeVarint(b, x)
}
func encodeUint64s(b *buffer, tag int, x []uint64) {
if len(x) > 2 {
// Use packed encoding
n1 := len(b.data)
for _, u := range x {
encodeVarint(b, u)
}
n2 := len(b.data)
encodeLength(b, tag, n2-n1)
n3 := len(b.data)
copy(b.tmp[:], b.data[n2:n3])
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
copy(b.data[n1:], b.tmp[:n3-n2])
return
}
for _, u := range x {
encodeUint64(b, tag, u)
}
}
func encodeUint64Opt(b *buffer, tag int, x uint64) {
if x == 0 {
return
}
encodeUint64(b, tag, x)
}
func encodeInt64(b *buffer, tag int, x int64) {
u := uint64(x)
encodeUint64(b, tag, u)
}
func encodeInt64Opt(b *buffer, tag int, x int64) {
if x == 0 {
return
}
encodeInt64(b, tag, x)
}
func encodeInt64s(b *buffer, tag int, x []int64) {
if len(x) > 2 {
// Use packed encoding
n1 := len(b.data)
for _, u := range x {
encodeVarint(b, uint64(u))
}
n2 := len(b.data)
encodeLength(b, tag, n2-n1)
n3 := len(b.data)
copy(b.tmp[:], b.data[n2:n3])
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
copy(b.data[n1:], b.tmp[:n3-n2])
return
}
for _, u := range x {
encodeInt64(b, tag, u)
}
}
func encodeString(b *buffer, tag int, x string) {
encodeLength(b, tag, len(x))
b.data = append(b.data, x...)
}
func encodeStrings(b *buffer, tag int, x []string) {
for _, s := range x {
encodeString(b, tag, s)
}
}
func encodeStringOpt(b *buffer, tag int, x string) {
if x == "" {
return
}
encodeString(b, tag, x)
}
func encodeBool(b *buffer, tag int, x bool) {
if x {
encodeUint64(b, tag, 1)
} else {
encodeUint64(b, tag, 0)
}
}
func encodeBoolOpt(b *buffer, tag int, x bool) {
if x == false {
return
}
encodeBool(b, tag, x)
}
func encodeMessage(b *buffer, tag int, m message) {
n1 := len(b.data)
m.encode(b)
n2 := len(b.data)
encodeLength(b, tag, n2-n1)
n3 := len(b.data)
copy(b.tmp[:], b.data[n2:n3])
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
copy(b.data[n1:], b.tmp[:n3-n2])
}
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package protopprof
import (
"fmt"
"os"
"runtime"
"strings"
"time"
"runtime/pprof/internal/profile"
)
// Copied from encoding/binary package, which can't be imported due to
// dependency cycles
// LittleEndian is the little-endian implementation of ByteOrder.
var lEndian littleEndian
// BigEndian is the big-endian implementation of ByteOrder.
var bEndian bigEndian
type littleEndian struct{}
type bigEndian struct{}
func (bigEndian) uint32(b []byte) uint32 {
_ = b[3] // bounds check hint to compiler; see golang.org/issue/14808
return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
}
func (bigEndian) uint64(b []byte) uint64 {
_ = b[7] // bounds check hint to compiler; see golang.org/issue/14808
return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
}
func (littleEndian) uint32(b []byte) uint32 {
_ = b[3] // bounds check hint to compiler; see golang.org/issue/14808
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func (littleEndian) uint64(b []byte) uint64 {
_ = b[7] // bounds check hint to compiler; see golang.org/issue/14808
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
func big32(b []byte) (uint64, []byte) {
if len(b) < 4 {
return 0, nil
}
return uint64(bEndian.uint32(b)), b[4:]
}
func little32(b []byte) (uint64, []byte) {
if len(b) < 4 {
return 0, nil
}
return uint64(lEndian.uint32(b)), b[4:]
}
func big64(b []byte) (uint64, []byte) {
if len(b) < 8 {
return 0, nil
}
return bEndian.uint64(b), b[8:]
}
func little64(b []byte) (uint64, []byte) {
if len(b) < 8 {
return 0, nil
}
return lEndian.uint64(b), b[8:]
}
// End of copy from encoding/binary package
type parser func([]byte) (uint64, []byte)
var parsers = []parser{
big32,
big64,
little32,
little64,
}
// parse returns a parsing function to parse native integers from a buffer.
func findParser(b []byte) parser {
for _, p := range parsers {
// If the second word decodes as 3, we have the right parser.
_, rest := p(b) // first word
n, _ := p(rest) // second word
if n == 3 {
return p
}
}
return nil
}
// decodeHeader parses binary CPU profiling stack trace data
// generated by runtime.CPUProfile() and returns the sample period,
// the rest of the profile and a parse function for parsing the profile. The
// function detects whether the legacy profile format is in little or big
// endian and whether it was generated by a 32-bit or 64-bit machine.
func decodeHeader(b []byte) (period uint64, parse parser, rest []byte, err error) {
const minRawProfile = 12 // Need a minimum of 3 words, at least 32-bit each.
if len(b) < minRawProfile {
return 0, nil, nil, fmt.Errorf("truncated raw profile: len %d", len(b))
}
if parse = findParser(b); parse == nil {
return 0, nil, nil, fmt.Errorf("cannot parse raw profile: header %v", b[:minRawProfile])
}
// skip 5-word header; 4th word is period
_, rest = parse(b)
_, rest = parse(rest)
_, rest = parse(rest)
period, rest = parse(rest)
_, rest = parse(rest)
if rest == nil {
return 0, nil, nil, fmt.Errorf("profile too short")
}
return period, parse, rest, nil
}
// translateCPUProfile parses binary CPU profiling stack trace data
// generated by runtime.CPUProfile() into a profile struct.
func TranslateCPUProfile(b []byte, startTime time.Time) (*profile.Profile, error) {
// Get the sample period from the header.
var n4 uint64
var getInt parser
var err error
n4, getInt, b, err = decodeHeader(b)
if err != nil {
return nil, err
}
// profile initialization taken from pprof tool
p := &profile.Profile{
Period: int64(n4) * 1000,
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
SampleType: []*profile.ValueType{
{Type: "samples", Unit: "count"},
{Type: "cpu", Unit: "nanoseconds"},
},
TimeNanos: int64(startTime.UnixNano()),
DurationNanos: time.Since(startTime).Nanoseconds(),
}
// Parse CPU samples from the profile.
locs := make(map[uint64]*profile.Location)
for len(b) > 0 {
var count, nstk uint64
count, b = getInt(b)
nstk, b = getInt(b)
if b == nil {
return nil, fmt.Errorf("unrecognized profile format")
}
var sloc []*profile.Location
addrs := make([]uint64, nstk)
for i := 0; i < int(nstk); i++ {
if b == nil {
return nil, fmt.Errorf("unrecognized profile format")
}
addrs[i], b = getInt(b)
}
// End of data marker, can return
if count == 0 && nstk == 1 && addrs[0] == 0 {
if runtime.GOOS == "linux" {
if err := addMappings(p); err != nil {
return nil, err
}
}
return p, nil
}
for i, addr := range addrs {
// Addresses from stack traces point to the next instruction after
// each call. Adjust by -1 to land somewhere on the actual call
// (except for the leaf, which is not a call).
if i > 0 {
addr--
}
loc := locs[addr]
if loc == nil {
loc = &profile.Location{
ID: uint64(len(p.Location) + 1),
Address: addr,
}
locs[addr] = loc
p.Location = append(p.Location, loc)
}
sloc = append(sloc, loc)
}
p.Sample = append(p.Sample, &profile.Sample{
Value: []int64{int64(count), int64(count) * int64(p.Period)},
Location: sloc,
})
}
return nil, fmt.Errorf("unrecognized profile format")
}
func addMappings(p *profile.Profile) error {
// Parse memory map from /proc/self/maps
f, err := os.Open("/proc/self/maps")
if err != nil {
return err
}
defer f.Close()
return p.ParseMemoryMap(f)
}
// Symbolization enables adding names to locations.
func Symbolize(p *profile.Profile) {
fns := profileFunctionMap{}
for _, l := range p.Location {
pc := uintptr(l.Address)
f := runtime.FuncForPC(pc)
if f == nil {
continue
}
file, lineno := f.FileLine(pc)
if l.Mapping != nil {
if f.Name() != "" {
l.Mapping.HasFunctions = true
}
if file != "" {
l.Mapping.HasFilenames = true
}
if lineno != 0 {
l.Mapping.HasLineNumbers = true
}
}
l.Line = []profile.Line{
{
Function: fns.findOrAddFunction(f.Name(), file, p),
Line: int64(lineno),
},
}
}
// Trim runtime functions. Always hide runtime.goexit. Other runtime
// functions are only hidden for heapz when they appear at the beginning.
isHeapz := p.PeriodType != nil && p.PeriodType.Type == "space"
for _, s := range p.Sample {
show := !isHeapz
var i int
for _, l := range s.Location {
if (len(l.Line) > 0) && (l.Line[0].Function != nil) {
name := l.Line[0].Function.Name
if (name == "runtime.goexit") || (!show && strings.HasPrefix(name, "runtime.")) {
continue
}
}
show = true
s.Location[i] = l
i++
}
s.Location = s.Location[:i]
}
}
type profileFunctionMap map[profile.Function]*profile.Function
func (fns profileFunctionMap) findOrAddFunction(name, filename string, p *profile.Profile) *profile.Function {
f := profile.Function{
Name: name,
SystemName: name,
Filename: filename,
}
if fp := fns[f]; fp != nil {
return fp
}
fp := new(profile.Function)
fns[f] = fp
*fp = f
fp.ID = uint64(len(p.Function) + 1)
p.Function = append(p.Function, fp)
return fp
}
func CleanupDuplicateLocations(p *profile.Profile) {
// The profile handler may duplicate the leaf frame, because it gets
// its address both from stack unwinding and from the signal
// context. Detect this and delete the duplicate, which has been
// adjusted by -1. The leaf address should not be adjusted as it is
// not a call.
for _, s := range p.Sample {
if len(s.Location) > 1 && s.Location[0].Address == s.Location[1].Address+1 {
s.Location = append(s.Location[:1], s.Location[2:]...)
}
}
}
...@@ -6,8 +6,8 @@ package pprof_test ...@@ -6,8 +6,8 @@ package pprof_test
import ( import (
"bytes" "bytes"
"fmt" "math"
"regexp" "reflect"
"runtime" "runtime"
. "runtime/pprof" . "runtime/pprof"
"testing" "testing"
...@@ -71,26 +71,48 @@ func TestMemoryProfiler(t *testing.T) { ...@@ -71,26 +71,48 @@ func TestMemoryProfiler(t *testing.T) {
memoryProfilerRun++ memoryProfilerRun++
tests := []string{ r := bytes.NewReader(buf.Bytes())
fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ p, err := Parse(r)
# 0x[0-9,a-f]+ runtime/pprof_test\.allocatePersistent1K\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:40 if err != nil {
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:63 t.Fatalf("can't parse pprof profile: %v", err)
`, 32*memoryProfilerRun, 1024*memoryProfilerRun, 32*memoryProfilerRun, 1024*memoryProfilerRun),
fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient1M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:21
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:61
`, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun),
fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient2M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:27
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:62
`, memoryProfilerRun, (2<<20)*memoryProfilerRun),
} }
if len(p.Sample) < 3 {
for _, test := range tests { t.Fatalf("few samples, got: %d", len(p.Sample))
if !regexp.MustCompile(test).Match(buf.Bytes()) { }
t.Fatalf("The entry did not match:\n%v\n\nProfile:\n%v\n", test, buf.String()) testSample := make(map[int][]int64)
testSample[0] = scaleHeapSample((int64)(32*memoryProfilerRun), (int64)(1024*memoryProfilerRun), p.Period)
testSample[0] = append(testSample[0], testSample[0][0], testSample[0][1])
testSample[1] = scaleHeapSample((int64)((1<<10)*memoryProfilerRun), (int64)((1<<20)*memoryProfilerRun), p.Period)
testSample[1] = append([]int64{0, 0}, testSample[1][0], testSample[1][1])
testSample[2] = scaleHeapSample((int64)(memoryProfilerRun), (int64)((2<<20)*memoryProfilerRun), p.Period)
testSample[2] = append([]int64{0, 0}, testSample[2][0], testSample[2][1])
for _, value := range testSample {
found := false
for i := range p.Sample {
if reflect.DeepEqual(p.Sample[i].Value, value) {
found = true
break
}
}
if !found {
t.Fatalf("the entry did not match any sample:\n%v\n", value)
} }
} }
} }
func scaleHeapSample(count, size, rate int64) []int64 {
if count == 0 || size == 0 {
return []int64{0, 0}
}
if rate <= 1 {
// if rate==1 all samples were collected so no adjustment is needed.
// if rate<1 treat as unknown and skip scaling.
return []int64{count, size}
}
avgSize := float64(size) / float64(count)
scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))
return []int64{int64(float64(count) * scale), int64(float64(size) * scale)}
}
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment