Skip to content

G
go
Commit 7ae27392 authored by Dmitry Vyukov's avatar Dmitry Vyukov
Browse files
Options

cmd/trace: split large traces into parts 

Trace viewer cannot handle traces larger than 256MB (limit on js string size):
https://github.com/catapult-project/catapult/issues/627
And even that is problematic (chrome hangs and crashes).
Split large traces into 100MB parts. Somewhat clumsy, but I don't see any other
solution (other than rewriting trace viewer). At least it works reliably now.

Fixes #15482

Change-Id: I993b5f43d22072c6f5bd041ab5888ce176f272b2
Reviewed-on: https://go-review.googlesource.com/22731Reviewed-by: default avatarHyang-Ah Hana Kim <hyangah@gmail.com>
parent ccf2c019
Hide whitespace changes
Inline Side-by-side
Showing with 116 additions and 20 deletions
src/cmd/trace/main.go
View file @ 7ae27392
......@@ -22,7 +22,9 @@ import (
"bufio"
"flag"
"fmt"
"html/template"
"internal/trace"
"log"
"net"
"net/http"
"os"
......@@ -76,20 +78,36 @@ func main() {
if err != nil {
dief("failed to create server socket: %v\n", err)
}
// Open browser.
log.Printf("Parsing trace...")
events, err := parseEvents()
if err != nil {
dief("%v\n", err)
}
log.Printf("Serializing trace...")
params := &traceParams{
events: events,
endTime: int64(1<<63 - 1),
}
data := generateTrace(params)
log.Printf("Splitting trace...")
ranges = splitTrace(data)
log.Printf("Opening browser")
if !startBrowser("http://" + ln.Addr().String()) {
fmt.Fprintf(os.Stderr, "Trace viewer is listening on http://%s\n", ln.Addr().String())
}
// Parse and symbolize trace asynchronously while browser opens.
go parseEvents()
// Start http server.
http.HandleFunc("/", httpMain)
err = http.Serve(ln, nil)
dief("failed to start http server: %v\n", err)
}
var ranges []Range
var loader struct {
once sync.Once
events []*trace.Event
......@@ -118,13 +136,23 @@ func parseEvents() ([]*trace.Event, error) {
// httpMain serves the starting page.
func httpMain(w http.ResponseWriter, r *http.Request) {
w.Write(templMain)
if err := templMain.Execute(w, ranges); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
var templMain = []byte(`
var templMain = template.Must(template.New("").Parse(`
<html>
<body>
<a href="/trace">View trace</a><br>
{{if $}}
{{range $e := $}}
<a href="/trace?start={{$e.Start}}&end={{$e.End}}">View trace ({{$e.Name}})</a><br>
{{end}}
<br>
{{else}}
<a href="/trace">View trace</a><br>
{{end}}
<a href="/goroutines">Goroutine analysis</a><br>
<a href="/io">Network blocking profile</a><br>
<a href="/block">Synchronization blocking profile</a><br>
......@@ -132,7 +160,7 @@ var templMain = []byte(`
<a href="/sched">Scheduler latency profile</a><br>
</body>
</html>
`)
`))
// startBrowser tries to open the URL in a browser
// and reports whether it succeeds.
......
src/cmd/trace/trace.go
View file @ 7ae27392
......@@ -14,6 +14,7 @@ import (
"runtime"
"strconv"
"strings"
"time"
)
func init() {
......@@ -29,17 +30,11 @@ func httpTrace(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
params := ""
if goids := r.FormValue("goid"); goids != "" {
goid, err := strconv.ParseUint(goids, 10, 64)
if err != nil {
http.Error(w, fmt.Sprintf("failed to parse goid parameter '%v': %v", goids, err), http.StatusInternalServerError)
return
}
params = fmt.Sprintf("?goid=%v", goid)
if err := r.ParseForm(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
html := strings.Replace(templTrace, "{{PARAMS}}", params, -1)
html := strings.Replace(templTrace, "{{PARAMS}}", r.Form.Encode(), -1)
w.Write([]byte(html))
}
......@@ -118,7 +113,7 @@ var templTrace = `
viewer.globalMode = true;
document.body.appendChild(viewer);
url = '/jsontrace{{PARAMS}}';
url = '/jsontrace?{{PARAMS}}';
load();
});
}());
......@@ -150,6 +145,7 @@ func httpJsonTrace(w http.ResponseWriter, r *http.Request) {
}
if goids := r.FormValue("goid"); goids != "" {
// If goid argument is present, we are rendering a trace for this particular goroutine.
goid, err := strconv.ParseUint(goids, 10, 64)
if err != nil {
log.Printf("failed to parse goid parameter '%v': %v", goids, err)
......@@ -164,13 +160,81 @@ func httpJsonTrace(w http.ResponseWriter, r *http.Request) {
params.gs = trace.RelatedGoroutines(events, goid)
}
err = json.NewEncoder(w).Encode(generateTrace(params))
data := generateTrace(params)
if startStr, endStr := r.FormValue("start"), r.FormValue("end"); startStr != "" && endStr != "" {
// If start/end arguments are present, we are rendering a range of the trace.
start, err := strconv.ParseUint(startStr, 10, 64)
if err != nil {
log.Printf("failed to parse start parameter '%v': %v", startStr, err)
return
}
end, err := strconv.ParseUint(endStr, 10, 64)
if err != nil {
log.Printf("failed to parse end parameter '%v': %v", endStr, err)
return
}
if start >= uint64(len(data.Events)) || end <= start || end > uint64(len(data.Events)) {
log.Printf("bogus start/end parameters: %v/%v, trace size %v", start, end, len(data.Events))
return
}
data.Events = append(data.Events[start:end], data.Events[data.footer:]...)
}
err = json.NewEncoder(w).Encode(data)
if err != nil {
log.Printf("failed to serialize trace: %v", err)
return
}
}
type Range struct {
Name string
Start int
End int
}
// splitTrace splits the trace into a number of ranges,
// each resulting in approx 100MB of json output (trace viewer can hardly handle more).
func splitTrace(data ViewerData) []Range {
const rangeSize = 100 << 20
var ranges []Range
cw := new(countingWriter)
enc := json.NewEncoder(cw)
// First calculate size of the mandatory part of the trace.
// This includes stack traces and thread names.
data1 := data
data1.Events = data.Events[data.footer:]
enc.Encode(data1)
auxSize := cw.size
cw.size = 0
// Then calculate size of each individual event and group them into ranges.
for i, start := 0, 0; i < data.footer; i++ {
enc.Encode(data.Events[i])
if cw.size+auxSize > rangeSize || i == data.footer-1 {
ranges = append(ranges, Range{
Name: fmt.Sprintf("%v-%v", time.Duration(data.Events[start].Time*1000), time.Duration(data.Events[i].Time*1000)),
Start: start,
End: i + 1,
})
start = i + 1
cw.size = 0
}
}
if len(ranges) == 1 {
ranges = nil
}
return ranges
}
type countingWriter struct {
size int
}
func (cw *countingWriter) Write(data []byte) (int, error) {
cw.size += len(data)
return len(data), nil
}
type traceParams struct {
events []*trace.Event
gtrace bool
......@@ -204,6 +268,9 @@ type ViewerData struct {
Events []*ViewerEvent `json:"traceEvents"`
Frames map[string]ViewerFrame `json:"stackFrames"`
TimeUnit string `json:"displayTimeUnit"`
// This is where mandatory part of the trace starts (e.g. thread names)
footer int
}
type ViewerEvent struct {
......@@ -355,6 +422,7 @@ func generateTrace(params *traceParams) ViewerData {
}
}
ctx.data.footer = len(ctx.data.Events)
ctx.emit(&ViewerEvent{Name: "process_name", Phase: "M", Pid: 0, Arg: &NameArg{"PROCS"}})
ctx.emit(&ViewerEvent{Name: "process_sort_index", Phase: "M", Pid: 0, Arg: &SortIndexArg{1}})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7