From 2c7758124d6a70a70c1893097388341481cc6cda Mon Sep 17 00:00:00 2001
From: wubin1989 <328454505@qq.com>
Date: Sat, 18 May 2024 00:59:01 +0800
Subject: [PATCH 1/3] optimize rest server
---
framework/rest/server.go | 90 ++++++++++++++++++++++++----------------
1 file changed, 54 insertions(+), 36 deletions(-)
diff --git a/framework/rest/server.go b/framework/rest/server.go
index 56d6ff96..5a2ab63b 100644
--- a/framework/rest/server.go
+++ b/framework/rest/server.go
@@ -9,6 +9,7 @@ import (
"github.com/klauspost/compress/gzhttp"
"github.com/olekukonko/tablewriter"
"github.com/rs/cors"
+ "github.com/samber/lo"
"github.com/unionj-cloud/go-doudou/v2/framework"
"github.com/unionj-cloud/go-doudou/v2/framework/config"
register "github.com/unionj-cloud/go-doudou/v2/framework/registry"
@@ -182,6 +183,7 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
fn(srv)
}
srv.middlewares = append(srv.middlewares,
+ srv.panicHandler,
tracing,
metrics,
gzipBody,
@@ -200,33 +202,7 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
requestid.RequestIDHandler,
handlers.ProxyHeaders,
)
- return srv
-}
-
-// AddRoute adds routes to router
-func (srv *RestServer) AddRoute(route ...Route) {
- srv.bizRoutes = append(srv.bizRoutes, route...)
-}
-
-// AddMiddleware adds middlewares to the end of chain
-func (srv *RestServer) AddMiddleware(mwf ...func(http.Handler) http.Handler) {
- for _, item := range mwf {
- srv.middlewares = append(srv.middlewares, item)
- }
-}
-
-// PreMiddleware adds middlewares to the head of chain
-func (srv *RestServer) PreMiddleware(mwf ...func(http.Handler) http.Handler) {
- var middlewares []MiddlewareFunc
- for _, item := range mwf {
- middlewares = append(middlewares, item)
- }
- srv.middlewares = append(middlewares, srv.middlewares...)
-}
-
-func (srv *RestServer) configure() {
if config.GddConfig.ManageEnable {
- srv.middlewares = append([]MiddlewareFunc{srv.panicHandler}, srv.middlewares...)
srv.middlewares = append([]MiddlewareFunc{PrometheusMiddleware}, srv.middlewares...)
gddRouter := srv.rootRouter.NewGroup(gddPathPrefix)
corsOpts := cors.New(cors.Options{
@@ -351,15 +327,6 @@ func (srv *RestServer) configure() {
}
debugRouter.Handler(item.Method, "/"+strings.TrimPrefix(item.Pattern, debugPathPrefix), h, item.Name)
}
- } else {
- srv.middlewares = append([]MiddlewareFunc{srv.panicHandler}, srv.middlewares...)
- }
- for _, item := range srv.bizRoutes {
- h := http.Handler(item.HandlerFunc)
- for i := len(srv.middlewares) - 1; i >= 0; i-- {
- h = srv.middlewares[i].Middleware(h)
- }
- srv.bizRouter.Handler(item.Method, item.Pattern, h, item.Name)
}
srv.rootRouter.NotFound = http.HandlerFunc(http.NotFound)
srv.rootRouter.MethodNotAllowed = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -370,6 +337,58 @@ func (srv *RestServer) configure() {
srv.rootRouter.NotFound = srv.middlewares[i].Middleware(srv.rootRouter.NotFound)
srv.rootRouter.MethodNotAllowed = srv.middlewares[i].Middleware(srv.rootRouter.MethodNotAllowed)
}
+ return srv
+}
+
+func (srv *RestServer) groupRoutes(routeGroup *httprouter.RouteGroup, routes []Route, mwf ...MiddlewareFunc) {
+ for _, item := range routes {
+ h := http.Handler(item.HandlerFunc)
+ for i := len(mwf) - 1; i >= 0; i-- {
+ h = mwf[i].Middleware(h)
+ }
+ routeGroup.Handler(item.Method, item.Pattern, h, item.Name)
+ srv.bizRoutes = append(srv.bizRoutes, item)
+ }
+}
+
+// AddRoute adds routes to router
+func (srv *RestServer) AddRoute(route ...Route) {
+ srv.groupRoutes(srv.bizRouter, route, srv.middlewares...)
+}
+
+// AddRoutes adds routes to router
+func (srv *RestServer) AddRoutes(routes []Route, mwf ...func(http.Handler) http.Handler) {
+ m := make([]MiddlewareFunc, 0)
+ m = append(m, srv.middlewares...)
+ m = append(m, lo.Map(mwf, func(item func(http.Handler) http.Handler, index int) MiddlewareFunc {
+ return item
+ })...)
+ srv.groupRoutes(srv.bizRouter, routes, m...)
+}
+
+// GroupRoutes adds routes to router
+func (srv *RestServer) GroupRoutes(group string, routes []Route, mwf ...func(http.Handler) http.Handler) {
+ m := make([]MiddlewareFunc, 0)
+ m = append(m, srv.middlewares...)
+ m = append(m, lo.Map(mwf, func(item func(http.Handler) http.Handler, index int) MiddlewareFunc {
+ return item
+ })...)
+ srv.groupRoutes(srv.bizRouter.NewGroup(group), routes, m...)
+}
+
+// AddMiddleware adds middlewares to the end of chain
+// Deprecated: use Use instead
+func (srv *RestServer) AddMiddleware(mwf ...func(http.Handler) http.Handler) {
+ for _, item := range mwf {
+ srv.middlewares = append(srv.middlewares, item)
+ }
+}
+
+// Use adds middlewares to the end of chain
+func (srv *RestServer) Use(mwf ...func(http.Handler) http.Handler) {
+ for _, item := range mwf {
+ srv.middlewares = append(srv.middlewares, item)
+ }
}
// Run runs http server
@@ -405,7 +424,6 @@ func (srv *RestServer) Serve(ln net.Listener) {
framework.PrintBanner()
framework.PrintLock.Lock()
register.NewRest(srv.data)
- srv.configure()
srv.printRoutes()
// Run our server in a goroutine so that it doesn't block.
go func() {
From 203af10a11f8c5accc1d3f82af14679330bf0207 Mon Sep 17 00:00:00 2001
From: wubin1989 <328454505@qq.com>
Date: Sun, 19 May 2024 19:22:25 +0800
Subject: [PATCH 2/3] optimize rest server
---
framework/rest/dochandler.go | 65 ++++++++++++++++++++++++++++++++----
framework/rest/server.go | 27 +--------------
2 files changed, 60 insertions(+), 32 deletions(-)
diff --git a/framework/rest/dochandler.go b/framework/rest/dochandler.go
index f933f2bd..6234c641 100644
--- a/framework/rest/dochandler.go
+++ b/framework/rest/dochandler.go
@@ -2,16 +2,34 @@ package rest
import (
"bytes"
+ "fmt"
+ "github.com/rs/cors"
"html/template"
"net/http"
+ "strings"
)
-// Oas store OpenAPI3.0 description json string
-var Oas string
+// window.docs = [
+// {
+// "label": "Banana",
+// "value": "http://localhost:6060/banana/go-doudou/doc",
+// },
+// {
+// "label": "Apple",
+// "value": "http://localhost:6060/apple/go-doudou/doc",
+// }
+//
+// ]
+var Docs []DocItem
var DocRoutes = docRoutes
-func docRoutes() []Route {
- return []Route{
+type DocItem struct {
+ Label string `json:"label"`
+ Value string `json:"value"`
+}
+
+func docRoutes(doc string) []Route {
+ routes := []Route{
{
Name: "GetDoc",
Method: "GET",
@@ -25,14 +43,15 @@ func docRoutes() []Route {
if tpl, err = template.New("onlinedoc.tmpl").Parse(onlinedocTmpl); err != nil {
panic(err)
}
- doc := Oas
docUrl := "openapi.json"
if err = tpl.Execute(&buf, struct {
Doc string
DocUrl string
+ Docs []DocItem
}{
Doc: doc,
DocUrl: docUrl,
+ Docs: Docs,
}); err != nil {
panic(err)
}
@@ -45,8 +64,42 @@ func docRoutes() []Route {
Method: "GET",
Pattern: "/go-doudou/openapi.json",
HandlerFunc: func(_writer http.ResponseWriter, _req *http.Request) {
- _writer.Write([]byte(Oas))
+ _writer.Write([]byte(doc))
},
},
}
+ corsOpts := cors.New(cors.Options{
+ AllowedMethods: []string{
+ http.MethodGet,
+ http.MethodPost,
+ http.MethodPut,
+ http.MethodPatch,
+ http.MethodDelete,
+ http.MethodOptions,
+ http.MethodHead,
+ },
+
+ AllowedHeaders: []string{
+ "*",
+ },
+
+ AllowOriginRequestFunc: func(r *http.Request, origin string) bool {
+ if strings.Contains(r.URL.Path, fmt.Sprintf("%sopenapi.json", gddPathPrefix)) {
+ return true
+ }
+ return false
+ },
+ })
+ gddmiddlewares := []MiddlewareFunc{corsOpts.Handler, MiddlewareFunc(basicAuth())}
+
+ for k, item := range routes {
+ h := http.Handler(item.HandlerFunc)
+ for i := len(gddmiddlewares) - 1; i >= 0; i-- {
+ h = gddmiddlewares[i].Middleware(h)
+ }
+ item.HandlerFunc = h.(http.HandlerFunc)
+ routes[k] = item
+ }
+
+ return routes
}
diff --git a/framework/rest/server.go b/framework/rest/server.go
index 5a2ab63b..3fbca718 100644
--- a/framework/rest/server.go
+++ b/framework/rest/server.go
@@ -2,13 +2,11 @@ package rest
import (
"context"
- "fmt"
"github.com/arl/statsviz"
"github.com/ascarter/requestid"
"github.com/gorilla/handlers"
"github.com/klauspost/compress/gzhttp"
"github.com/olekukonko/tablewriter"
- "github.com/rs/cors"
"github.com/samber/lo"
"github.com/unionj-cloud/go-doudou/v2/framework"
"github.com/unionj-cloud/go-doudou/v2/framework/config"
@@ -205,31 +203,8 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
if config.GddConfig.ManageEnable {
srv.middlewares = append([]MiddlewareFunc{PrometheusMiddleware}, srv.middlewares...)
gddRouter := srv.rootRouter.NewGroup(gddPathPrefix)
- corsOpts := cors.New(cors.Options{
- AllowedMethods: []string{
- http.MethodGet,
- http.MethodPost,
- http.MethodPut,
- http.MethodPatch,
- http.MethodDelete,
- http.MethodOptions,
- http.MethodHead,
- },
-
- AllowedHeaders: []string{
- "*",
- },
-
- AllowOriginRequestFunc: func(r *http.Request, origin string) bool {
- if r.URL.Path == fmt.Sprintf("%sopenapi.json", gddPathPrefix) {
- return true
- }
- return false
- },
- })
basicAuthMiddle := MiddlewareFunc(basicAuth())
- gddmiddlewares := []MiddlewareFunc{metrics, corsOpts.Handler, basicAuthMiddle}
- srv.gddRoutes = append(srv.gddRoutes, docRoutes()...)
+ gddmiddlewares := []MiddlewareFunc{metrics, basicAuthMiddle}
srv.gddRoutes = append(srv.gddRoutes, promRoutes()...)
srv.gddRoutes = append(srv.gddRoutes, configRoutes()...)
if _, ok := config.ServiceDiscoveryMap()[constants.SD_MEMBERLIST]; ok {
From 9c72280199655a31405388520dfa5886a584c5c9 Mon Sep 17 00:00:00 2001
From: wubin1989 <328454505@qq.com>
Date: Mon, 20 May 2024 01:28:38 +0800
Subject: [PATCH 3/3] optimize rest server
---
cmd/internal/svc/codegen/init.go | 3 +-
cmd/internal/svc/codegen/plugin.go | 4 +-
cmd/internal/svc/parser/parser.go | 6 +-
cmd/internal/templates/mainmain.go | 1 +
cmd/internal/templates/plugin.go | 4 +-
framework/rest/dochandler.go | 10 +-
framework/rest/docindex.go | 2 +-
framework/rest/httprouter/routegroup.go | 10 +-
framework/rest/lazyregexp/lazyre.go | 78 ++
framework/rest/pprof.go | 465 ++++++++
framework/rest/profile/encode.go | 482 ++++++++
framework/rest/profile/filter.go | 158 +++
framework/rest/profile/legacy_profile.go | 1268 ++++++++++++++++++++++
framework/rest/profile/merge.go | 461 ++++++++
framework/rest/profile/profile.go | 622 +++++++++++
framework/rest/profile/profile_test.go | 64 ++
framework/rest/profile/proto.go | 356 ++++++
framework/rest/profile/proto_test.go | 71 ++
framework/rest/profile/prune.go | 97 ++
framework/rest/server.go | 81 +-
go.mod | 8 +-
go.sum | 13 +-
version/version.go | 2 +-
23 files changed, 4208 insertions(+), 58 deletions(-)
create mode 100644 framework/rest/lazyregexp/lazyre.go
create mode 100644 framework/rest/pprof.go
create mode 100644 framework/rest/profile/encode.go
create mode 100644 framework/rest/profile/filter.go
create mode 100644 framework/rest/profile/legacy_profile.go
create mode 100644 framework/rest/profile/merge.go
create mode 100644 framework/rest/profile/profile.go
create mode 100644 framework/rest/profile/profile_test.go
create mode 100644 framework/rest/profile/proto.go
create mode 100644 framework/rest/profile/proto_test.go
create mode 100644 framework/rest/profile/prune.go
diff --git a/cmd/internal/svc/codegen/init.go b/cmd/internal/svc/codegen/init.go
index cd31fa75..122b2b3d 100644
--- a/cmd/internal/svc/codegen/init.go
+++ b/cmd/internal/svc/codegen/init.go
@@ -16,6 +16,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "strings"
"text/template"
)
@@ -301,7 +302,7 @@ func InitProj(conf InitProjConfig) {
SvcName string
Version string
}{
- DtoPackage: filepath.Join(modName, "dto"),
+ DtoPackage: strings.ReplaceAll(filepath.Join(modName, "dto"), string(os.PathSeparator), "/"),
SvcName: svcName,
Version: version.Release,
})
diff --git a/cmd/internal/svc/codegen/plugin.go b/cmd/internal/svc/codegen/plugin.go
index 06923dbe..62eab053 100644
--- a/cmd/internal/svc/codegen/plugin.go
+++ b/cmd/internal/svc/codegen/plugin.go
@@ -32,7 +32,9 @@ func genPlugin(dir string, ic astutils.InterfaceCollector) {
}
defer f.Close()
- if tpl, err = template.New(templates.PluginTmpl).Parse(templates.PluginTmpl); err != nil {
+ funcMap := make(map[string]interface{})
+ funcMap["toLower"] = strings.ToLower
+ if tpl, err = template.New(templates.PluginTmpl).Funcs(funcMap).Parse(templates.PluginTmpl); err != nil {
panic(err)
}
diff --git a/cmd/internal/svc/parser/parser.go b/cmd/internal/svc/parser/parser.go
index e6146bdb..84b2b6aa 100644
--- a/cmd/internal/svc/parser/parser.go
+++ b/cmd/internal/svc/parser/parser.go
@@ -366,11 +366,7 @@ func pathsOf(ic astutils.InterfaceCollector, config GenDocConfig) map[string]v3.
var gofileTmpl = `package {{.SvcPackage}}
-import "github.com/unionj-cloud/go-doudou/v2/framework/rest"
-
-func init() {
- rest.Oas = ` + "`" + `{{.Doc}}` + "`" + `
-}
+var Oas = ` + "`" + `{{.Doc}}` + "`" + `
`
type GenDocConfig struct {
diff --git a/cmd/internal/templates/mainmain.go b/cmd/internal/templates/mainmain.go
index b1ce290c..c02aa54b 100644
--- a/cmd/internal/templates/mainmain.go
+++ b/cmd/internal/templates/mainmain.go
@@ -57,6 +57,7 @@ func main() {
go func() {
grpcServer.RunWithPipe(lis)
}()
+ srv.AddRoutes(rest.DocRoutes(""))
srv.Run()
}
`
diff --git a/cmd/internal/templates/plugin.go b/cmd/internal/templates/plugin.go
index e6358376..7d8378b6 100644
--- a/cmd/internal/templates/plugin.go
+++ b/cmd/internal/templates/plugin.go
@@ -58,7 +58,9 @@ func (receiver *{{.SvcName}}Plugin) Initialize(restServer *rest.RestServer, grpc
conf := config.LoadFromEnv()
svc := {{.ServiceAlias}}.New{{.SvcName}}(conf)
receiver.serviceInstance = svc
- restServer.AddRoute(httpsrv.Routes(httpsrv.New{{.SvcName}}Handler(svc))...)
+ routes := httpsrv.Routes(httpsrv.New{{.SvcName}}Handler(svc))
+ restServer.GroupRoutes("/{{.SvcName | toLower}}", routes)
+ restServer.GroupRoutes("/{{.SvcName | toLower}}", rest.DocRoutes(service.Oas))
if grpcServer.Server == nil {
grpcServer.Server = grpc.NewServer(
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
diff --git a/framework/rest/dochandler.go b/framework/rest/dochandler.go
index 6234c641..9bd92fe7 100644
--- a/framework/rest/dochandler.go
+++ b/framework/rest/dochandler.go
@@ -2,8 +2,10 @@ package rest
import (
"bytes"
+ "encoding/json"
"fmt"
"github.com/rs/cors"
+ "github.com/unionj-cloud/go-doudou/v2/toolkit/stringutils"
"html/template"
"net/http"
"strings"
@@ -29,6 +31,9 @@ type DocItem struct {
}
func docRoutes(doc string) []Route {
+ if stringutils.IsEmpty(doc) {
+ doc = "{}"
+ }
routes := []Route{
{
Name: "GetDoc",
@@ -44,14 +49,15 @@ func docRoutes(doc string) []Route {
panic(err)
}
docUrl := "openapi.json"
+ docs, _ := json.Marshal(Docs)
if err = tpl.Execute(&buf, struct {
Doc string
DocUrl string
- Docs []DocItem
+ Docs string
}{
Doc: doc,
DocUrl: docUrl,
- Docs: Docs,
+ Docs: string(docs),
}); err != nil {
panic(err)
}
diff --git a/framework/rest/docindex.go b/framework/rest/docindex.go
index 6210090c..ce937826 100644
--- a/framework/rest/docindex.go
+++ b/framework/rest/docindex.go
@@ -1,3 +1,3 @@
package rest
-var onlinedocTmpl = "Vue Typescript Admin\n"
+var onlinedocTmpl = "Vue Typescript Admin"
diff --git a/framework/rest/httprouter/routegroup.go b/framework/rest/httprouter/routegroup.go
index 721e1d59..cc4ff92f 100644
--- a/framework/rest/httprouter/routegroup.go
+++ b/framework/rest/httprouter/routegroup.go
@@ -30,19 +30,19 @@ func newRouteGroup(r *Router, path string) *RouteGroup {
}
func (r *RouteGroup) NewGroup(path string) *RouteGroup {
- return newRouteGroup(r.r, r.subPath(path))
+ return newRouteGroup(r.r, r.SubPath(path))
}
func (r *RouteGroup) Handle(method, path string, handle Handle, name ...string) {
- r.r.Handle(method, r.subPath(path), handle, name...)
+ r.r.Handle(method, r.SubPath(path), handle, name...)
}
func (r *RouteGroup) Handler(method, path string, handler http.Handler, name ...string) {
- r.r.Handler(method, r.subPath(path), handler, name...)
+ r.r.Handler(method, r.SubPath(path), handler, name...)
}
func (r *RouteGroup) HandlerFunc(method, path string, handler http.HandlerFunc, name ...string) {
- r.r.HandlerFunc(method, r.subPath(path), handler, name...)
+ r.r.HandlerFunc(method, r.SubPath(path), handler, name...)
}
func (r *RouteGroup) GET(path string, handle Handle) {
@@ -67,7 +67,7 @@ func (r *RouteGroup) DELETE(path string, handle Handle) {
r.Handle("DELETE", path, handle)
}
-func (r *RouteGroup) subPath(path string) string {
+func (r *RouteGroup) SubPath(path string) string {
result := r.p + path
if stringutils.IsEmpty(result) {
return "/"
diff --git a/framework/rest/lazyregexp/lazyre.go b/framework/rest/lazyregexp/lazyre.go
new file mode 100644
index 00000000..2681af35
--- /dev/null
+++ b/framework/rest/lazyregexp/lazyre.go
@@ -0,0 +1,78 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package lazyregexp is a thin wrapper over regexp, allowing the use of global
+// regexp variables without forcing them to be compiled at init.
+package lazyregexp
+
+import (
+ "os"
+ "regexp"
+ "strings"
+ "sync"
+)
+
+// Regexp is a wrapper around regexp.Regexp, where the underlying regexp will be
+// compiled the first time it is needed.
+type Regexp struct {
+ str string
+ once sync.Once
+ rx *regexp.Regexp
+}
+
+func (r *Regexp) re() *regexp.Regexp {
+ r.once.Do(r.build)
+ return r.rx
+}
+
+func (r *Regexp) build() {
+ r.rx = regexp.MustCompile(r.str)
+ r.str = ""
+}
+
+func (r *Regexp) FindSubmatch(s []byte) [][]byte {
+ return r.re().FindSubmatch(s)
+}
+
+func (r *Regexp) FindStringSubmatch(s string) []string {
+ return r.re().FindStringSubmatch(s)
+}
+
+func (r *Regexp) FindStringSubmatchIndex(s string) []int {
+ return r.re().FindStringSubmatchIndex(s)
+}
+
+func (r *Regexp) ReplaceAllString(src, repl string) string {
+ return r.re().ReplaceAllString(src, repl)
+}
+
+func (r *Regexp) FindString(s string) string {
+ return r.re().FindString(s)
+}
+
+func (r *Regexp) FindAllString(s string, n int) []string {
+ return r.re().FindAllString(s, n)
+}
+
+func (r *Regexp) MatchString(s string) bool {
+ return r.re().MatchString(s)
+}
+
+func (r *Regexp) SubexpNames() []string {
+ return r.re().SubexpNames()
+}
+
+var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test")
+
+// New creates a new lazy regexp, delaying the compiling work until it is first
+// needed. If the code is being run as part of tests, the regexp compiling will
+// happen immediately.
+func New(str string) *Regexp {
+ lr := &Regexp{str: str}
+ if inTest {
+ // In tests, always compile the regexps early.
+ lr.re()
+ }
+ return lr
+}
diff --git a/framework/rest/pprof.go b/framework/rest/pprof.go
new file mode 100644
index 00000000..0348a1a1
--- /dev/null
+++ b/framework/rest/pprof.go
@@ -0,0 +1,465 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package rest serves via its HTTP server runtime profiling data
+// in the format expected by the pprof visualization tool.
+//
+// The package is typically only imported for the side effect of
+// registering its HTTP handlers.
+// The handled paths all begin with /debug/pprof/.
+//
+// To use pprof, link this package into your program:
+//
+// import _ "net/http/pprof"
+//
+// If your application is not already running an http server, you
+// need to start one. Add "net/http" and "log" to your imports and
+// the following code to your main function:
+//
+// go func() {
+// log2.Println(http.ListenAndServe("localhost:6060", nil))
+// }()
+//
+// By default, all the profiles listed in [runtime/pprof.Profile] are
+// available (via [Handler]), in addition to the [Cmdline], [Profile], [Symbol],
+// and [Trace] profiles defined in this package.
+// If you are not using DefaultServeMux, you will have to register handlers
+// with the mux you are using.
+//
+// # Parameters
+//
+// Parameters can be passed via GET query params:
+//
+// - debug=N (all profiles): response format: N = 0: binary (default), N > 0: plaintext
+// - gc=N (heap profile): N > 0: run a garbage collection cycle before profiling
+// - seconds=N (allocs, block, goroutine, heap, mutex, threadcreate profiles): return a delta profile
+// - seconds=N (cpu (profile), trace profiles): profile for the given duration
+//
+// # Usage examples
+//
+// Use the pprof tool to look at the heap profile:
+//
+// go tool pprof http://localhost:6060/debug/pprof/heap
+//
+// Or to look at a 30-second CPU profile:
+//
+// go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
+//
+// Or to look at the goroutine blocking profile, after calling
+// [runtime.SetBlockProfileRate] in your program:
+//
+// go tool pprof http://localhost:6060/debug/pprof/block
+//
+// Or to look at the holders of contended mutexes, after calling
+// [runtime.SetMutexProfileFraction] in your program:
+//
+// go tool pprof http://localhost:6060/debug/pprof/mutex
+//
+// The package also exports a handler that serves execution trace data
+// for the "go tool trace" command. To collect a 5-second execution trace:
+//
+// curl -o trace.out http://localhost:6060/debug/pprof/trace?seconds=5
+// go tool trace trace.out
+//
+// To view all available profiles, open http://localhost:6060/debug/pprof/
+// in your browser.
+//
+// For a study of the facility in action, visit
+// https://blog.golang.org/2011/06/profiling-go-programs.html.
+package rest
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "fmt"
+ "github.com/unionj-cloud/go-doudou/v2/framework/config"
+ "github.com/unionj-cloud/go-doudou/v2/framework/rest/profile"
+ "html"
+ "io"
+ log2 "log"
+ "net/http"
+ "net/url"
+ "os"
+ "runtime"
+ "runtime/pprof"
+ "runtime/trace"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+func init() {
+ http.HandleFunc(config.GddConfig.RouteRootPath+"/debug/pprof/", Index)
+ http.HandleFunc(config.GddConfig.RouteRootPath+"/debug/pprof/cmdline", Cmdline)
+ http.HandleFunc(config.GddConfig.RouteRootPath+"/debug/pprof/profile", Profile)
+ http.HandleFunc(config.GddConfig.RouteRootPath+"/debug/pprof/symbol", Symbol)
+ http.HandleFunc(config.GddConfig.RouteRootPath+"/debug/pprof/trace", Trace)
+}
+
+// Cmdline responds with the running program's
+// command line, with arguments separated by NUL bytes.
+// The package initialization registers it as /debug/pprof/cmdline.
+func Cmdline(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ fmt.Fprint(w, strings.Join(os.Args, "\x00"))
+}
+
+func sleep(r *http.Request, d time.Duration) {
+ select {
+ case <-time.After(d):
+ case <-r.Context().Done():
+ }
+}
+
+func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool {
+ srv, ok := r.Context().Value(http.ServerContextKey).(*http.Server)
+ return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds()
+}
+
+func serveError(w http.ResponseWriter, status int, txt string) {
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ w.Header().Set("X-Go-Pprof", "1")
+ w.Header().Del("Content-Disposition")
+ w.WriteHeader(status)
+ fmt.Fprintln(w, txt)
+}
+
+// Profile responds with the pprof-formatted cpu profile.
+// Profiling lasts for duration specified in seconds GET parameter, or for 30 seconds if not specified.
+// The package initialization registers it as /debug/pprof/profile.
+func Profile(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ sec, err := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
+ if sec <= 0 || err != nil {
+ sec = 30
+ }
+
+ if durationExceedsWriteTimeout(r, float64(sec)) {
+ serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+ return
+ }
+
+ // Set Content Type assuming StartCPUProfile will work,
+ // because if it does it starts writing.
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Disposition", `attachment; filename="profile"`)
+ if err := pprof.StartCPUProfile(w); err != nil {
+ // StartCPUProfile failed, so no writes yet.
+ serveError(w, http.StatusInternalServerError,
+ fmt.Sprintf("Could not enable CPU profiling: %s", err))
+ return
+ }
+ sleep(r, time.Duration(sec)*time.Second)
+ pprof.StopCPUProfile()
+}
+
+// Trace responds with the execution trace in binary form.
+// Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified.
+// The package initialization registers it as /debug/pprof/trace.
+func Trace(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64)
+ if sec <= 0 || err != nil {
+ sec = 1
+ }
+
+ if durationExceedsWriteTimeout(r, sec) {
+ serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+ return
+ }
+
+ // Set Content Type assuming trace.Start will work,
+ // because if it does it starts writing.
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Disposition", `attachment; filename="trace"`)
+ if err := trace.Start(w); err != nil {
+ // trace.Start failed, so no writes yet.
+ serveError(w, http.StatusInternalServerError,
+ fmt.Sprintf("Could not enable tracing: %s", err))
+ return
+ }
+ sleep(r, time.Duration(sec*float64(time.Second)))
+ trace.Stop()
+}
+
+// Symbol looks up the program counters listed in the request,
+// responding with a table mapping program counters to function names.
+// The package initialization registers it as /debug/pprof/symbol.
+func Symbol(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+ // We have to read the whole POST body before
+ // writing any output. Buffer the output here.
+ var buf bytes.Buffer
+
+ // We don't know how many symbols we have, but we
+ // do have symbol information. Pprof only cares whether
+ // this number is 0 (no symbols available) or > 0.
+ fmt.Fprintf(&buf, "num_symbols: 1\n")
+
+ var b *bufio.Reader
+ if r.Method == "POST" {
+ b = bufio.NewReader(r.Body)
+ } else {
+ b = bufio.NewReader(strings.NewReader(r.URL.RawQuery))
+ }
+
+ for {
+ word, err := b.ReadSlice('+')
+ if err == nil {
+ word = word[0 : len(word)-1] // trim +
+ }
+ pc, _ := strconv.ParseUint(string(word), 0, 64)
+ if pc != 0 {
+ f := runtime.FuncForPC(uintptr(pc))
+ if f != nil {
+ fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name())
+ }
+ }
+
+ // Wait until here to check for err; the last
+ // symbol will have an err because it doesn't end in +.
+ if err != nil {
+ if err != io.EOF {
+ fmt.Fprintf(&buf, "reading request: %v\n", err)
+ }
+ break
+ }
+ }
+
+ w.Write(buf.Bytes())
+}
+
+// Handler returns an HTTP handler that serves the named profile.
+// Available profiles can be found in [runtime/pprof.Profile].
+func Handler(name string) http.Handler {
+ return handler(name)
+}
+
+type handler string
+
+func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ p := pprof.Lookup(string(name))
+ if p == nil {
+ serveError(w, http.StatusNotFound, "Unknown profile")
+ return
+ }
+ if sec := r.FormValue("seconds"); sec != "" {
+ name.serveDeltaProfile(w, r, p, sec)
+ return
+ }
+ gc, _ := strconv.Atoi(r.FormValue("gc"))
+ if name == "heap" && gc > 0 {
+ runtime.GC()
+ }
+ debug, _ := strconv.Atoi(r.FormValue("debug"))
+ if debug != 0 {
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ } else {
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
+ }
+ p.WriteTo(w, debug)
+}
+
+func (name handler) serveDeltaProfile(w http.ResponseWriter, r *http.Request, p *pprof.Profile, secStr string) {
+ sec, err := strconv.ParseInt(secStr, 10, 64)
+ if err != nil || sec <= 0 {
+ serveError(w, http.StatusBadRequest, `invalid value for "seconds" - must be a positive integer`)
+ return
+ }
+ if !profileSupportsDelta[name] {
+ serveError(w, http.StatusBadRequest, `"seconds" parameter is not supported for this profile type`)
+ return
+ }
+ // 'name' should be a key in profileSupportsDelta.
+ if durationExceedsWriteTimeout(r, float64(sec)) {
+ serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+ return
+ }
+ debug, _ := strconv.Atoi(r.FormValue("debug"))
+ if debug != 0 {
+ serveError(w, http.StatusBadRequest, "seconds and debug params are incompatible")
+ return
+ }
+ p0, err := collectProfile(p)
+ if err != nil {
+ serveError(w, http.StatusInternalServerError, "failed to collect profile")
+ return
+ }
+
+ t := time.NewTimer(time.Duration(sec) * time.Second)
+ defer t.Stop()
+
+ select {
+ case <-r.Context().Done():
+ err := r.Context().Err()
+ if err == context.DeadlineExceeded {
+ serveError(w, http.StatusRequestTimeout, err.Error())
+ } else { // TODO: what's a good status code for canceled requests? 400?
+ serveError(w, http.StatusInternalServerError, err.Error())
+ }
+ return
+ case <-t.C:
+ }
+
+ p1, err := collectProfile(p)
+ if err != nil {
+ serveError(w, http.StatusInternalServerError, "failed to collect profile")
+ return
+ }
+ ts := p1.TimeNanos
+ dur := p1.TimeNanos - p0.TimeNanos
+
+ p0.Scale(-1)
+
+ p1, err = profile.Merge([]*profile.Profile{p0, p1})
+ if err != nil {
+ serveError(w, http.StatusInternalServerError, "failed to compute delta")
+ return
+ }
+
+ p1.TimeNanos = ts // set since we don't know what profile.Merge set for TimeNanos.
+ p1.DurationNanos = dur
+
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s-delta"`, name))
+ p1.Write(w)
+}
+
+func collectProfile(p *pprof.Profile) (*profile.Profile, error) {
+ var buf bytes.Buffer
+ if err := p.WriteTo(&buf, 0); err != nil {
+ return nil, err
+ }
+ ts := time.Now().UnixNano()
+ p0, err := profile.Parse(&buf)
+ if err != nil {
+ return nil, err
+ }
+ p0.TimeNanos = ts
+ return p0, nil
+}
+
+var profileSupportsDelta = map[handler]bool{
+ "allocs": true,
+ "block": true,
+ "goroutine": true,
+ "heap": true,
+ "mutex": true,
+ "threadcreate": true,
+}
+
+var profileDescriptions = map[string]string{
+ "allocs": "A sampling of all past memory allocations",
+ "block": "Stack traces that led to blocking on synchronization primitives",
+ "cmdline": "The command line invocation of the current program",
+ "goroutine": "Stack traces of all current goroutines. Use debug=2 as a query parameter to export in the same format as an unrecovered panic.",
+ "heap": "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.",
+ "mutex": "Stack traces of holders of contended mutexes",
+ "profile": "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.",
+ "threadcreate": "Stack traces that led to the creation of new OS threads",
+ "trace": "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.",
+}
+
+type profileEntry struct {
+ Name string
+ Href string
+ Desc string
+ Count int
+}
+
+// Index responds with the pprof-formatted profile named by the request.
+// For example, config.GddConfig.RouteRootPath + "/debug/pprof/heap" serves the "heap" profile.
+// Index responds to a request for config.GddConfig.RouteRootPath + "/debug/pprof/" with an HTML page
+// listing the available profiles.
+func Index(w http.ResponseWriter, r *http.Request) {
+ if name, found := strings.CutPrefix(r.URL.Path, config.GddConfig.RouteRootPath+"/debug/pprof/"); found {
+ if name != "" {
+ handler(name).ServeHTTP(w, r)
+ return
+ }
+ }
+
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+
+ var profiles []profileEntry
+ for _, p := range pprof.Profiles() {
+ profiles = append(profiles, profileEntry{
+ Name: p.Name(),
+ Href: p.Name(),
+ Desc: profileDescriptions[p.Name()],
+ Count: p.Count(),
+ })
+ }
+
+ // Adding other profiles exposed from within this package
+ for _, p := range []string{"cmdline", "profile", "trace"} {
+ profiles = append(profiles, profileEntry{
+ Name: p,
+ Href: p,
+ Desc: profileDescriptions[p],
+ })
+ }
+
+ sort.Slice(profiles, func(i, j int) bool {
+ return profiles[i].Name < profiles[j].Name
+ })
+
+ if err := indexTmplExecute(w, profiles); err != nil {
+ log2.Print(err)
+ }
+}
+
+func indexTmplExecute(w io.Writer, profiles []profileEntry) error {
+ var b bytes.Buffer
+ b.WriteString(`
+
+/debug/pprof/
+
+
+
+/debug/pprof/
+
+Set debug=1 as a query parameter to export in legacy text format
+
+Types of profiles available:
+
+| Count | Profile |
+`)
+
+ for _, profile := range profiles {
+ link := &url.URL{Path: profile.Href, RawQuery: "debug=1"}
+ fmt.Fprintf(&b, "| %d | %s |
\n", profile.Count, link, html.EscapeString(profile.Name))
+ }
+
+ b.WriteString(`
+full goroutine stack dump
+
+
+Profile Descriptions:
+
+`)
+ for _, profile := range profiles {
+ fmt.Fprintf(&b, "%s:
%s
+
+
+`)
+
+ _, err := w.Write(b.Bytes())
+ return err
+}
diff --git a/framework/rest/profile/encode.go b/framework/rest/profile/encode.go
new file mode 100644
index 00000000..94d04bf0
--- /dev/null
+++ b/framework/rest/profile/encode.go
@@ -0,0 +1,482 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package profile
+
+import (
+ "errors"
+ "fmt"
+ "sort"
+)
+
+func (p *Profile) decoder() []decoder {
+ return profileDecoder
+}
+
+// preEncode populates the unexported fields to be used by encode
+// (with suffix X) from the corresponding exported fields. The
+// exported fields are cleared up to facilitate testing.
+func (p *Profile) preEncode() {
+ strings := make(map[string]int)
+ addString(strings, "")
+
+ for _, st := range p.SampleType {
+ st.typeX = addString(strings, st.Type)
+ st.unitX = addString(strings, st.Unit)
+ }
+
+ for _, s := range p.Sample {
+ s.labelX = nil
+ var keys []string
+ for k := range s.Label {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ vs := s.Label[k]
+ for _, v := range vs {
+ s.labelX = append(s.labelX,
+ Label{
+ keyX: addString(strings, k),
+ strX: addString(strings, v),
+ },
+ )
+ }
+ }
+ var numKeys []string
+ for k := range s.NumLabel {
+ numKeys = append(numKeys, k)
+ }
+ sort.Strings(numKeys)
+ for _, k := range numKeys {
+ vs := s.NumLabel[k]
+ for _, v := range vs {
+ s.labelX = append(s.labelX,
+ Label{
+ keyX: addString(strings, k),
+ numX: v,
+ },
+ )
+ }
+ }
+ s.locationIDX = nil
+ for _, l := range s.Location {
+ s.locationIDX = append(s.locationIDX, l.ID)
+ }
+ }
+
+ for _, m := range p.Mapping {
+ m.fileX = addString(strings, m.File)
+ m.buildIDX = addString(strings, m.BuildID)
+ }
+
+ for _, l := range p.Location {
+ for i, ln := range l.Line {
+ if ln.Function != nil {
+ l.Line[i].functionIDX = ln.Function.ID
+ } else {
+ l.Line[i].functionIDX = 0
+ }
+ }
+ if l.Mapping != nil {
+ l.mappingIDX = l.Mapping.ID
+ } else {
+ l.mappingIDX = 0
+ }
+ }
+ for _, f := range p.Function {
+ f.nameX = addString(strings, f.Name)
+ f.systemNameX = addString(strings, f.SystemName)
+ f.filenameX = addString(strings, f.Filename)
+ }
+
+ p.dropFramesX = addString(strings, p.DropFrames)
+ p.keepFramesX = addString(strings, p.KeepFrames)
+
+ if pt := p.PeriodType; pt != nil {
+ pt.typeX = addString(strings, pt.Type)
+ pt.unitX = addString(strings, pt.Unit)
+ }
+
+ p.stringTable = make([]string, len(strings))
+ for s, i := range strings {
+ p.stringTable[i] = s
+ }
+}
+
+func (p *Profile) encode(b *buffer) {
+ for _, x := range p.SampleType {
+ encodeMessage(b, 1, x)
+ }
+ for _, x := range p.Sample {
+ encodeMessage(b, 2, x)
+ }
+ for _, x := range p.Mapping {
+ encodeMessage(b, 3, x)
+ }
+ for _, x := range p.Location {
+ encodeMessage(b, 4, x)
+ }
+ for _, x := range p.Function {
+ encodeMessage(b, 5, x)
+ }
+ encodeStrings(b, 6, p.stringTable)
+ encodeInt64Opt(b, 7, p.dropFramesX)
+ encodeInt64Opt(b, 8, p.keepFramesX)
+ encodeInt64Opt(b, 9, p.TimeNanos)
+ encodeInt64Opt(b, 10, p.DurationNanos)
+ if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) {
+ encodeMessage(b, 11, p.PeriodType)
+ }
+ encodeInt64Opt(b, 12, p.Period)
+}
+
+var profileDecoder = []decoder{
+ nil, // 0
+ // repeated ValueType sample_type = 1
+ func(b *buffer, m message) error {
+ x := new(ValueType)
+ pp := m.(*Profile)
+ pp.SampleType = append(pp.SampleType, x)
+ return decodeMessage(b, x)
+ },
+ // repeated Sample sample = 2
+ func(b *buffer, m message) error {
+ x := new(Sample)
+ pp := m.(*Profile)
+ pp.Sample = append(pp.Sample, x)
+ return decodeMessage(b, x)
+ },
+ // repeated Mapping mapping = 3
+ func(b *buffer, m message) error {
+ x := new(Mapping)
+ pp := m.(*Profile)
+ pp.Mapping = append(pp.Mapping, x)
+ return decodeMessage(b, x)
+ },
+ // repeated Location location = 4
+ func(b *buffer, m message) error {
+ x := new(Location)
+ pp := m.(*Profile)
+ pp.Location = append(pp.Location, x)
+ return decodeMessage(b, x)
+ },
+ // repeated Function function = 5
+ func(b *buffer, m message) error {
+ x := new(Function)
+ pp := m.(*Profile)
+ pp.Function = append(pp.Function, x)
+ return decodeMessage(b, x)
+ },
+ // repeated string string_table = 6
+ func(b *buffer, m message) error {
+ err := decodeStrings(b, &m.(*Profile).stringTable)
+ if err != nil {
+ return err
+ }
+ if m.(*Profile).stringTable[0] != "" {
+ return errors.New("string_table[0] must be ''")
+ }
+ return nil
+ },
+ // repeated int64 drop_frames = 7
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).dropFramesX) },
+ // repeated int64 keep_frames = 8
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).keepFramesX) },
+ // repeated int64 time_nanos = 9
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).TimeNanos) },
+ // repeated int64 duration_nanos = 10
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).DurationNanos) },
+ // optional string period_type = 11
+ func(b *buffer, m message) error {
+ x := new(ValueType)
+ pp := m.(*Profile)
+ pp.PeriodType = x
+ return decodeMessage(b, x)
+ },
+ // repeated int64 period = 12
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).Period) },
+ // repeated int64 comment = 13
+ func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Profile).commentX) },
+ // int64 defaultSampleType = 14
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).defaultSampleTypeX) },
+}
+
+// postDecode takes the unexported fields populated by decode (with
+// suffix X) and populates the corresponding exported fields.
+// The unexported fields are cleared up to facilitate testing.
+func (p *Profile) postDecode() error {
+ var err error
+
+ mappings := make(map[uint64]*Mapping)
+ for _, m := range p.Mapping {
+ m.File, err = getString(p.stringTable, &m.fileX, err)
+ m.BuildID, err = getString(p.stringTable, &m.buildIDX, err)
+ mappings[m.ID] = m
+ }
+
+ functions := make(map[uint64]*Function)
+ for _, f := range p.Function {
+ f.Name, err = getString(p.stringTable, &f.nameX, err)
+ f.SystemName, err = getString(p.stringTable, &f.systemNameX, err)
+ f.Filename, err = getString(p.stringTable, &f.filenameX, err)
+ functions[f.ID] = f
+ }
+
+ locations := make(map[uint64]*Location)
+ for _, l := range p.Location {
+ l.Mapping = mappings[l.mappingIDX]
+ l.mappingIDX = 0
+ for i, ln := range l.Line {
+ if id := ln.functionIDX; id != 0 {
+ l.Line[i].Function = functions[id]
+ if l.Line[i].Function == nil {
+ return fmt.Errorf("Function ID %d not found", id)
+ }
+ l.Line[i].functionIDX = 0
+ }
+ }
+ locations[l.ID] = l
+ }
+
+ for _, st := range p.SampleType {
+ st.Type, err = getString(p.stringTable, &st.typeX, err)
+ st.Unit, err = getString(p.stringTable, &st.unitX, err)
+ }
+
+ for _, s := range p.Sample {
+ labels := make(map[string][]string)
+ numLabels := make(map[string][]int64)
+ for _, l := range s.labelX {
+ var key, value string
+ key, err = getString(p.stringTable, &l.keyX, err)
+ if l.strX != 0 {
+ value, err = getString(p.stringTable, &l.strX, err)
+ labels[key] = append(labels[key], value)
+ } else {
+ numLabels[key] = append(numLabels[key], l.numX)
+ }
+ }
+ if len(labels) > 0 {
+ s.Label = labels
+ }
+ if len(numLabels) > 0 {
+ s.NumLabel = numLabels
+ }
+ s.Location = nil
+ for _, lid := range s.locationIDX {
+ s.Location = append(s.Location, locations[lid])
+ }
+ s.locationIDX = nil
+ }
+
+ p.DropFrames, err = getString(p.stringTable, &p.dropFramesX, err)
+ p.KeepFrames, err = getString(p.stringTable, &p.keepFramesX, err)
+
+ if pt := p.PeriodType; pt == nil {
+ p.PeriodType = &ValueType{}
+ }
+
+ if pt := p.PeriodType; pt != nil {
+ pt.Type, err = getString(p.stringTable, &pt.typeX, err)
+ pt.Unit, err = getString(p.stringTable, &pt.unitX, err)
+ }
+ for _, i := range p.commentX {
+ var c string
+ c, err = getString(p.stringTable, &i, err)
+ p.Comments = append(p.Comments, c)
+ }
+
+ p.commentX = nil
+ p.DefaultSampleType, err = getString(p.stringTable, &p.defaultSampleTypeX, err)
+ p.stringTable = nil
+ return err
+}
+
+func (p *ValueType) decoder() []decoder {
+ return valueTypeDecoder
+}
+
+func (p *ValueType) encode(b *buffer) {
+ encodeInt64Opt(b, 1, p.typeX)
+ encodeInt64Opt(b, 2, p.unitX)
+}
+
+var valueTypeDecoder = []decoder{
+ nil, // 0
+ // optional int64 type = 1
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).typeX) },
+ // optional int64 unit = 2
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).unitX) },
+}
+
+func (p *Sample) decoder() []decoder {
+ return sampleDecoder
+}
+
+func (p *Sample) encode(b *buffer) {
+ encodeUint64s(b, 1, p.locationIDX)
+ for _, x := range p.Value {
+ encodeInt64(b, 2, x)
+ }
+ for _, x := range p.labelX {
+ encodeMessage(b, 3, x)
+ }
+}
+
+var sampleDecoder = []decoder{
+ nil, // 0
+ // repeated uint64 location = 1
+ func(b *buffer, m message) error { return decodeUint64s(b, &m.(*Sample).locationIDX) },
+ // repeated int64 value = 2
+ func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Sample).Value) },
+ // repeated Label label = 3
+ func(b *buffer, m message) error {
+ s := m.(*Sample)
+ n := len(s.labelX)
+ s.labelX = append(s.labelX, Label{})
+ return decodeMessage(b, &s.labelX[n])
+ },
+}
+
+func (p Label) decoder() []decoder {
+ return labelDecoder
+}
+
+func (p Label) encode(b *buffer) {
+ encodeInt64Opt(b, 1, p.keyX)
+ encodeInt64Opt(b, 2, p.strX)
+ encodeInt64Opt(b, 3, p.numX)
+}
+
+var labelDecoder = []decoder{
+ nil, // 0
+ // optional int64 key = 1
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).keyX) },
+ // optional int64 str = 2
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).strX) },
+ // optional int64 num = 3
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).numX) },
+}
+
+func (p *Mapping) decoder() []decoder {
+ return mappingDecoder
+}
+
+func (p *Mapping) encode(b *buffer) {
+ encodeUint64Opt(b, 1, p.ID)
+ encodeUint64Opt(b, 2, p.Start)
+ encodeUint64Opt(b, 3, p.Limit)
+ encodeUint64Opt(b, 4, p.Offset)
+ encodeInt64Opt(b, 5, p.fileX)
+ encodeInt64Opt(b, 6, p.buildIDX)
+ encodeBoolOpt(b, 7, p.HasFunctions)
+ encodeBoolOpt(b, 8, p.HasFilenames)
+ encodeBoolOpt(b, 9, p.HasLineNumbers)
+ encodeBoolOpt(b, 10, p.HasInlineFrames)
+}
+
+var mappingDecoder = []decoder{
+ nil, // 0
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).ID) }, // optional uint64 id = 1
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Start) }, // optional uint64 memory_offset = 2
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Limit) }, // optional uint64 memory_limit = 3
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Offset) }, // optional uint64 file_offset = 4
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).fileX) }, // optional int64 filename = 5
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).buildIDX) }, // optional int64 build_id = 6
+ func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFunctions) }, // optional bool has_functions = 7
+ func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFilenames) }, // optional bool has_filenames = 8
+ func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasLineNumbers) }, // optional bool has_line_numbers = 9
+ func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasInlineFrames) }, // optional bool has_inline_frames = 10
+}
+
+func (p *Location) decoder() []decoder {
+ return locationDecoder
+}
+
+func (p *Location) encode(b *buffer) {
+ encodeUint64Opt(b, 1, p.ID)
+ encodeUint64Opt(b, 2, p.mappingIDX)
+ encodeUint64Opt(b, 3, p.Address)
+ for i := range p.Line {
+ encodeMessage(b, 4, &p.Line[i])
+ }
+}
+
+var locationDecoder = []decoder{
+ nil, // 0
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).ID) }, // optional uint64 id = 1;
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).mappingIDX) }, // optional uint64 mapping_id = 2;
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).Address) }, // optional uint64 address = 3;
+ func(b *buffer, m message) error { // repeated Line line = 4
+ pp := m.(*Location)
+ n := len(pp.Line)
+ pp.Line = append(pp.Line, Line{})
+ return decodeMessage(b, &pp.Line[n])
+ },
+}
+
+func (p *Line) decoder() []decoder {
+ return lineDecoder
+}
+
+func (p *Line) encode(b *buffer) {
+ encodeUint64Opt(b, 1, p.functionIDX)
+ encodeInt64Opt(b, 2, p.Line)
+}
+
+var lineDecoder = []decoder{
+ nil, // 0
+ // optional uint64 function_id = 1
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Line).functionIDX) },
+ // optional int64 line = 2
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Line).Line) },
+}
+
+func (p *Function) decoder() []decoder {
+ return functionDecoder
+}
+
+func (p *Function) encode(b *buffer) {
+ encodeUint64Opt(b, 1, p.ID)
+ encodeInt64Opt(b, 2, p.nameX)
+ encodeInt64Opt(b, 3, p.systemNameX)
+ encodeInt64Opt(b, 4, p.filenameX)
+ encodeInt64Opt(b, 5, p.StartLine)
+}
+
+var functionDecoder = []decoder{
+ nil, // 0
+ // optional uint64 id = 1
+ func(b *buffer, m message) error { return decodeUint64(b, &m.(*Function).ID) },
+ // optional int64 function_name = 2
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).nameX) },
+ // optional int64 function_system_name = 3
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).systemNameX) },
+ // repeated int64 filename = 4
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).filenameX) },
+ // optional int64 start_line = 5
+ func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).StartLine) },
+}
+
+func addString(strings map[string]int, s string) int64 {
+ i, ok := strings[s]
+ if !ok {
+ i = len(strings)
+ strings[s] = i
+ }
+ return int64(i)
+}
+
+func getString(strings []string, strng *int64, err error) (string, error) {
+ if err != nil {
+ return "", err
+ }
+ s := int(*strng)
+ if s < 0 || s >= len(strings) {
+ return "", errMalformed
+ }
+ *strng = 0
+ return strings[s], nil
+}
diff --git a/framework/rest/profile/filter.go b/framework/rest/profile/filter.go
new file mode 100644
index 00000000..141dd1f4
--- /dev/null
+++ b/framework/rest/profile/filter.go
@@ -0,0 +1,158 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Implements methods to filter samples from profiles.
+
+package profile
+
+import "regexp"
+
+// FilterSamplesByName filters the samples in a profile and only keeps
+// samples where at least one frame matches focus but none match ignore.
+// Returns true is the corresponding regexp matched at least one sample.
+func (p *Profile) FilterSamplesByName(focus, ignore, hide *regexp.Regexp) (fm, im, hm bool) {
+ focusOrIgnore := make(map[uint64]bool)
+ hidden := make(map[uint64]bool)
+ for _, l := range p.Location {
+ if ignore != nil && l.matchesName(ignore) {
+ im = true
+ focusOrIgnore[l.ID] = false
+ } else if focus == nil || l.matchesName(focus) {
+ fm = true
+ focusOrIgnore[l.ID] = true
+ }
+ if hide != nil && l.matchesName(hide) {
+ hm = true
+ l.Line = l.unmatchedLines(hide)
+ if len(l.Line) == 0 {
+ hidden[l.ID] = true
+ }
+ }
+ }
+
+ s := make([]*Sample, 0, len(p.Sample))
+ for _, sample := range p.Sample {
+ if focusedAndNotIgnored(sample.Location, focusOrIgnore) {
+ if len(hidden) > 0 {
+ var locs []*Location
+ for _, loc := range sample.Location {
+ if !hidden[loc.ID] {
+ locs = append(locs, loc)
+ }
+ }
+ if len(locs) == 0 {
+ // Remove sample with no locations (by not adding it to s).
+ continue
+ }
+ sample.Location = locs
+ }
+ s = append(s, sample)
+ }
+ }
+ p.Sample = s
+
+ return
+}
+
+// matchesName reports whether the function name or file in the
+// location matches the regular expression.
+func (loc *Location) matchesName(re *regexp.Regexp) bool {
+ for _, ln := range loc.Line {
+ if fn := ln.Function; fn != nil {
+ if re.MatchString(fn.Name) {
+ return true
+ }
+ if re.MatchString(fn.Filename) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// unmatchedLines returns the lines in the location that do not match
+// the regular expression.
+func (loc *Location) unmatchedLines(re *regexp.Regexp) []Line {
+ var lines []Line
+ for _, ln := range loc.Line {
+ if fn := ln.Function; fn != nil {
+ if re.MatchString(fn.Name) {
+ continue
+ }
+ if re.MatchString(fn.Filename) {
+ continue
+ }
+ }
+ lines = append(lines, ln)
+ }
+ return lines
+}
+
+// focusedAndNotIgnored looks up a slice of ids against a map of
+// focused/ignored locations. The map only contains locations that are
+// explicitly focused or ignored. Returns whether there is at least
+// one focused location but no ignored locations.
+func focusedAndNotIgnored(locs []*Location, m map[uint64]bool) bool {
+ var f bool
+ for _, loc := range locs {
+ if focus, focusOrIgnore := m[loc.ID]; focusOrIgnore {
+ if focus {
+ // Found focused location. Must keep searching in case there
+ // is an ignored one as well.
+ f = true
+ } else {
+ // Found ignored location. Can return false right away.
+ return false
+ }
+ }
+ }
+ return f
+}
+
+// TagMatch selects tags for filtering
+type TagMatch func(key, val string, nval int64) bool
+
+// FilterSamplesByTag removes all samples from the profile, except
+// those that match focus and do not match the ignore regular
+// expression.
+func (p *Profile) FilterSamplesByTag(focus, ignore TagMatch) (fm, im bool) {
+ samples := make([]*Sample, 0, len(p.Sample))
+ for _, s := range p.Sample {
+ focused, ignored := focusedSample(s, focus, ignore)
+ fm = fm || focused
+ im = im || ignored
+ if focused && !ignored {
+ samples = append(samples, s)
+ }
+ }
+ p.Sample = samples
+ return
+}
+
+// focusedSample checks a sample against focus and ignore regexps.
+// Returns whether the focus/ignore regexps match any tags.
+func focusedSample(s *Sample, focus, ignore TagMatch) (fm, im bool) {
+ fm = focus == nil
+ for key, vals := range s.Label {
+ for _, val := range vals {
+ if ignore != nil && ignore(key, val, 0) {
+ im = true
+ }
+ if !fm && focus(key, val, 0) {
+ fm = true
+ }
+ }
+ }
+ for key, vals := range s.NumLabel {
+ for _, val := range vals {
+ if ignore != nil && ignore(key, "", val) {
+ im = true
+ }
+ if !fm && focus(key, "", val) {
+ fm = true
+ }
+ }
+ }
+ return fm, im
+}
diff --git a/framework/rest/profile/legacy_profile.go b/framework/rest/profile/legacy_profile.go
new file mode 100644
index 00000000..a16c2906
--- /dev/null
+++ b/framework/rest/profile/legacy_profile.go
@@ -0,0 +1,1268 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements parsers to convert legacy profiles into the
+// profile.proto format.
+
+package profile
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "github.com/unionj-cloud/go-doudou/v2/framework/rest/lazyregexp"
+ "io"
+ "math"
+ "strconv"
+ "strings"
+)
+
+var (
+ countStartRE = lazyregexp.New(`\A(\w+) profile: total \d+\n\z`)
+ countRE = lazyregexp.New(`\A(\d+) @(( 0x[0-9a-f]+)+)\n\z`)
+
+ heapHeaderRE = lazyregexp.New(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`)
+ heapSampleRE = lazyregexp.New(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`)
+
+ contentionSampleRE = lazyregexp.New(`(\d+) *(\d+) @([ x0-9a-f]*)`)
+
+ hexNumberRE = lazyregexp.New(`0x[0-9a-f]+`)
+
+ growthHeaderRE = lazyregexp.New(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ growthz`)
+
+ fragmentationHeaderRE = lazyregexp.New(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz`)
+
+ threadzStartRE = lazyregexp.New(`--- threadz \d+ ---`)
+ threadStartRE = lazyregexp.New(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`)
+
+ procMapsRE = lazyregexp.New(`([[:xdigit:]]+)-([[:xdigit:]]+)\s+([-rwxp]+)\s+([[:xdigit:]]+)\s+([[:xdigit:]]+):([[:xdigit:]]+)\s+([[:digit:]]+)\s*(\S+)?`)
+
+ briefMapsRE = lazyregexp.New(`\s*([[:xdigit:]]+)-([[:xdigit:]]+):\s*(\S+)(\s.*@)?([[:xdigit:]]+)?`)
+
+ // LegacyHeapAllocated instructs the heapz parsers to use the
+ // allocated memory stats instead of the default in-use memory. Note
+ // that tcmalloc doesn't provide all allocated memory, only in-use
+ // stats.
+ LegacyHeapAllocated bool
+)
+
+func isSpaceOrComment(line string) bool {
+ trimmed := strings.TrimSpace(line)
+ return len(trimmed) == 0 || trimmed[0] == '#'
+}
+
+// parseGoCount parses a Go count profile (e.g., threadcreate or
+// goroutine) and returns a new Profile.
+func parseGoCount(b []byte) (*Profile, error) {
+ r := bytes.NewBuffer(b)
+
+ var line string
+ var err error
+ for {
+ // Skip past comments and empty lines seeking a real header.
+ line, err = r.ReadString('\n')
+ if err != nil {
+ return nil, err
+ }
+ if !isSpaceOrComment(line) {
+ break
+ }
+ }
+
+ m := countStartRE.FindStringSubmatch(line)
+ if m == nil {
+ return nil, errUnrecognized
+ }
+ profileType := m[1]
+ p := &Profile{
+ PeriodType: &ValueType{Type: profileType, Unit: "count"},
+ Period: 1,
+ SampleType: []*ValueType{{Type: profileType, Unit: "count"}},
+ }
+ locations := make(map[uint64]*Location)
+ for {
+ line, err = r.ReadString('\n')
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if isSpaceOrComment(line) {
+ continue
+ }
+ if strings.HasPrefix(line, "---") {
+ break
+ }
+ m := countRE.FindStringSubmatch(line)
+ if m == nil {
+ return nil, errMalformed
+ }
+ n, err := strconv.ParseInt(m[1], 0, 64)
+ if err != nil {
+ return nil, errMalformed
+ }
+ fields := strings.Fields(m[2])
+ locs := make([]*Location, 0, len(fields))
+ for _, stk := range fields {
+ addr, err := strconv.ParseUint(stk, 0, 64)
+ if err != nil {
+ return nil, errMalformed
+ }
+ // Adjust all frames by -1 to land on the call instruction.
+ addr--
+ loc := locations[addr]
+ if loc == nil {
+ loc = &Location{
+ Address: addr,
+ }
+ locations[addr] = loc
+ p.Location = append(p.Location, loc)
+ }
+ locs = append(locs, loc)
+ }
+ p.Sample = append(p.Sample, &Sample{
+ Location: locs,
+ Value: []int64{n},
+ })
+ }
+
+ if err = parseAdditionalSections(strings.TrimSpace(line), r, p); err != nil {
+ return nil, err
+ }
+ return p, nil
+}
+
+// remapLocationIDs ensures there is a location for each address
+// referenced by a sample, and remaps the samples to point to the new
+// location ids.
+func (p *Profile) remapLocationIDs() {
+ seen := make(map[*Location]bool, len(p.Location))
+ var locs []*Location
+
+ for _, s := range p.Sample {
+ for _, l := range s.Location {
+ if seen[l] {
+ continue
+ }
+ l.ID = uint64(len(locs) + 1)
+ locs = append(locs, l)
+ seen[l] = true
+ }
+ }
+ p.Location = locs
+}
+
+func (p *Profile) remapFunctionIDs() {
+ seen := make(map[*Function]bool, len(p.Function))
+ var fns []*Function
+
+ for _, l := range p.Location {
+ for _, ln := range l.Line {
+ fn := ln.Function
+ if fn == nil || seen[fn] {
+ continue
+ }
+ fn.ID = uint64(len(fns) + 1)
+ fns = append(fns, fn)
+ seen[fn] = true
+ }
+ }
+ p.Function = fns
+}
+
+// remapMappingIDs matches location addresses with existing mappings
+// and updates them appropriately. This is O(N*M), if this ever shows
+// up as a bottleneck, evaluate sorting the mappings and doing a
+// binary search, which would make it O(N*log(M)).
+func (p *Profile) remapMappingIDs() {
+ if len(p.Mapping) == 0 {
+ return
+ }
+
+ // Some profile handlers will incorrectly set regions for the main
+ // executable if its section is remapped. Fix them through heuristics.
+
+ // Remove the initial mapping if named '/anon_hugepage' and has a
+ // consecutive adjacent mapping.
+ if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") {
+ if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start {
+ p.Mapping = p.Mapping[1:]
+ }
+ }
+
+ for _, l := range p.Location {
+ if a := l.Address; a != 0 {
+ for _, m := range p.Mapping {
+ if m.Start <= a && a < m.Limit {
+ l.Mapping = m
+ break
+ }
+ }
+ }
+ }
+
+ // Reset all mapping IDs.
+ for i, m := range p.Mapping {
+ m.ID = uint64(i + 1)
+ }
+}
+
+var cpuInts = []func([]byte) (uint64, []byte){
+ get32l,
+ get32b,
+ get64l,
+ get64b,
+}
+
+func get32l(b []byte) (uint64, []byte) {
+ if len(b) < 4 {
+ return 0, nil
+ }
+ return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24, b[4:]
+}
+
+func get32b(b []byte) (uint64, []byte) {
+ if len(b) < 4 {
+ return 0, nil
+ }
+ return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24, b[4:]
+}
+
+func get64l(b []byte) (uint64, []byte) {
+ if len(b) < 8 {
+ return 0, nil
+ }
+ return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:]
+}
+
+func get64b(b []byte) (uint64, []byte) {
+ if len(b) < 8 {
+ return 0, nil
+ }
+ return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:]
+}
+
+// ParseTracebacks parses a set of tracebacks and returns a newly
+// populated profile. It will accept any text file and generate a
+// Profile out of it with any hex addresses it can identify, including
+// a process map if it can recognize one. Each sample will include a
+// tag "source" with the addresses recognized in string format.
+func ParseTracebacks(b []byte) (*Profile, error) {
+ r := bytes.NewBuffer(b)
+
+ p := &Profile{
+ PeriodType: &ValueType{Type: "trace", Unit: "count"},
+ Period: 1,
+ SampleType: []*ValueType{
+ {Type: "trace", Unit: "count"},
+ },
+ }
+
+ var sources []string
+ var sloc []*Location
+
+ locs := make(map[uint64]*Location)
+ for {
+ l, err := r.ReadString('\n')
+ if err != nil {
+ if err != io.EOF {
+ return nil, err
+ }
+ if l == "" {
+ break
+ }
+ }
+ if sectionTrigger(l) == memoryMapSection {
+ break
+ }
+ if s, addrs := extractHexAddresses(l); len(s) > 0 {
+ for _, addr := range addrs {
+ // Addresses from stack traces point to the next instruction after
+ // each call. Adjust by -1 to land somewhere on the actual call.
+ addr--
+ loc := locs[addr]
+ if locs[addr] == nil {
+ loc = &Location{
+ Address: addr,
+ }
+ p.Location = append(p.Location, loc)
+ locs[addr] = loc
+ }
+ sloc = append(sloc, loc)
+ }
+
+ sources = append(sources, s...)
+ } else {
+ if len(sources) > 0 || len(sloc) > 0 {
+ addTracebackSample(sloc, sources, p)
+ sloc, sources = nil, nil
+ }
+ }
+ }
+
+ // Add final sample to save any leftover data.
+ if len(sources) > 0 || len(sloc) > 0 {
+ addTracebackSample(sloc, sources, p)
+ }
+
+ if err := p.ParseMemoryMap(r); err != nil {
+ return nil, err
+ }
+ return p, nil
+}
+
+func addTracebackSample(l []*Location, s []string, p *Profile) {
+ p.Sample = append(p.Sample,
+ &Sample{
+ Value: []int64{1},
+ Location: l,
+ Label: map[string][]string{"source": s},
+ })
+}
+
+// parseCPU parses a profilez legacy profile and returns a newly
+// populated Profile.
+//
+// The general format for profilez samples is a sequence of words in
+// binary format. The first words are a header with the following data:
+//
+// 1st word -- 0
+// 2nd word -- 3
+// 3rd word -- 0 if a c++ application, 1 if a java application.
+// 4th word -- Sampling period (in microseconds).
+// 5th word -- Padding.
+func parseCPU(b []byte) (*Profile, error) {
+ var parse func([]byte) (uint64, []byte)
+ var n1, n2, n3, n4, n5 uint64
+ for _, parse = range cpuInts {
+ var tmp []byte
+ n1, tmp = parse(b)
+ n2, tmp = parse(tmp)
+ n3, tmp = parse(tmp)
+ n4, tmp = parse(tmp)
+ n5, tmp = parse(tmp)
+
+ if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 == 0 {
+ b = tmp
+ return cpuProfile(b, int64(n4), parse)
+ }
+ }
+ return nil, errUnrecognized
+}
+
+// cpuProfile returns a new Profile from C++ profilez data.
+// b is the profile bytes after the header, period is the profiling
+// period, and parse is a function to parse 8-byte chunks from the
+// profile in its native endianness.
+func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) {
+ p := &Profile{
+ Period: period * 1000,
+ PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"},
+ SampleType: []*ValueType{
+ {Type: "samples", Unit: "count"},
+ {Type: "cpu", Unit: "nanoseconds"},
+ },
+ }
+ var err error
+ if b, _, err = parseCPUSamples(b, parse, true, p); err != nil {
+ return nil, err
+ }
+
+ // If all samples have the same second-to-the-bottom frame, it
+ // strongly suggests that it is an uninteresting artifact of
+ // measurement -- a stack frame pushed by the signal handler. The
+ // bottom frame is always correct as it is picked up from the signal
+ // structure, not the stack. Check if this is the case and if so,
+ // remove.
+ if len(p.Sample) > 1 && len(p.Sample[0].Location) > 1 {
+ allSame := true
+ id1 := p.Sample[0].Location[1].Address
+ for _, s := range p.Sample {
+ if len(s.Location) < 2 || id1 != s.Location[1].Address {
+ allSame = false
+ break
+ }
+ }
+ if allSame {
+ for _, s := range p.Sample {
+ s.Location = append(s.Location[:1], s.Location[2:]...)
+ }
+ }
+ }
+
+ if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil {
+ return nil, err
+ }
+ return p, nil
+}
+
+// parseCPUSamples parses a collection of profilez samples from a
+// profile.
+//
+// profilez samples are a repeated sequence of stack frames of the
+// form:
+//
+// 1st word -- The number of times this stack was encountered.
+// 2nd word -- The size of the stack (StackSize).
+// 3rd word -- The first address on the stack.
+// ...
+// StackSize + 2 -- The last address on the stack
+//
+// The last stack trace is of the form:
+//
+// 1st word -- 0
+// 2nd word -- 1
+// 3rd word -- 0
+//
+// Addresses from stack traces may point to the next instruction after
+// each call. Optionally adjust by -1 to land somewhere on the actual
+// call (except for the leaf, which is not a call).
+func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust bool, p *Profile) ([]byte, map[uint64]*Location, error) {
+ locs := make(map[uint64]*Location)
+ for len(b) > 0 {
+ var count, nstk uint64
+ count, b = parse(b)
+ nstk, b = parse(b)
+ if b == nil || nstk > uint64(len(b)/4) {
+ return nil, nil, errUnrecognized
+ }
+ var sloc []*Location
+ addrs := make([]uint64, nstk)
+ for i := 0; i < int(nstk); i++ {
+ addrs[i], b = parse(b)
+ }
+
+ if count == 0 && nstk == 1 && addrs[0] == 0 {
+ // End of data marker
+ break
+ }
+ for i, addr := range addrs {
+ if adjust && i > 0 {
+ addr--
+ }
+ loc := locs[addr]
+ if loc == nil {
+ loc = &Location{
+ Address: addr,
+ }
+ locs[addr] = loc
+ p.Location = append(p.Location, loc)
+ }
+ sloc = append(sloc, loc)
+ }
+ p.Sample = append(p.Sample,
+ &Sample{
+ Value: []int64{int64(count), int64(count) * p.Period},
+ Location: sloc,
+ })
+ }
+ // Reached the end without finding the EOD marker.
+ return b, locs, nil
+}
+
+// parseHeap parses a heapz legacy or a growthz profile and
+// returns a newly populated Profile.
+func parseHeap(b []byte) (p *Profile, err error) {
+ r := bytes.NewBuffer(b)
+ l, err := r.ReadString('\n')
+ if err != nil {
+ return nil, errUnrecognized
+ }
+
+ sampling := ""
+
+ if header := heapHeaderRE.FindStringSubmatch(l); header != nil {
+ p = &Profile{
+ SampleType: []*ValueType{
+ {Type: "objects", Unit: "count"},
+ {Type: "space", Unit: "bytes"},
+ },
+ PeriodType: &ValueType{Type: "objects", Unit: "bytes"},
+ }
+
+ var period int64
+ if len(header[6]) > 0 {
+ if period, err = strconv.ParseInt(header[6], 10, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ }
+
+ switch header[5] {
+ case "heapz_v2", "heap_v2":
+ sampling, p.Period = "v2", period
+ case "heapprofile":
+ sampling, p.Period = "", 1
+ case "heap":
+ sampling, p.Period = "v2", period/2
+ default:
+ return nil, errUnrecognized
+ }
+ } else if header = growthHeaderRE.FindStringSubmatch(l); header != nil {
+ p = &Profile{
+ SampleType: []*ValueType{
+ {Type: "objects", Unit: "count"},
+ {Type: "space", Unit: "bytes"},
+ },
+ PeriodType: &ValueType{Type: "heapgrowth", Unit: "count"},
+ Period: 1,
+ }
+ } else if header = fragmentationHeaderRE.FindStringSubmatch(l); header != nil {
+ p = &Profile{
+ SampleType: []*ValueType{
+ {Type: "objects", Unit: "count"},
+ {Type: "space", Unit: "bytes"},
+ },
+ PeriodType: &ValueType{Type: "allocations", Unit: "count"},
+ Period: 1,
+ }
+ } else {
+ return nil, errUnrecognized
+ }
+
+ if LegacyHeapAllocated {
+ for _, st := range p.SampleType {
+ st.Type = "alloc_" + st.Type
+ }
+ } else {
+ for _, st := range p.SampleType {
+ st.Type = "inuse_" + st.Type
+ }
+ }
+
+ locs := make(map[uint64]*Location)
+ for {
+ l, err = r.ReadString('\n')
+ if err != nil {
+ if err != io.EOF {
+ return nil, err
+ }
+
+ if l == "" {
+ break
+ }
+ }
+
+ if isSpaceOrComment(l) {
+ continue
+ }
+ l = strings.TrimSpace(l)
+
+ if sectionTrigger(l) != unrecognizedSection {
+ break
+ }
+
+ value, blocksize, addrs, err := parseHeapSample(l, p.Period, sampling)
+ if err != nil {
+ return nil, err
+ }
+ var sloc []*Location
+ for _, addr := range addrs {
+ // Addresses from stack traces point to the next instruction after
+ // each call. Adjust by -1 to land somewhere on the actual call.
+ addr--
+ loc := locs[addr]
+ if locs[addr] == nil {
+ loc = &Location{
+ Address: addr,
+ }
+ p.Location = append(p.Location, loc)
+ locs[addr] = loc
+ }
+ sloc = append(sloc, loc)
+ }
+
+ p.Sample = append(p.Sample, &Sample{
+ Value: value,
+ Location: sloc,
+ NumLabel: map[string][]int64{"bytes": {blocksize}},
+ })
+ }
+
+ if err = parseAdditionalSections(l, r, p); err != nil {
+ return nil, err
+ }
+ return p, nil
+}
+
+// parseHeapSample parses a single row from a heap profile into a new Sample.
+func parseHeapSample(line string, rate int64, sampling string) (value []int64, blocksize int64, addrs []uint64, err error) {
+ sampleData := heapSampleRE.FindStringSubmatch(line)
+ if len(sampleData) != 6 {
+ return value, blocksize, addrs, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData))
+ }
+
+ // Use first two values by default; tcmalloc sampling generates the
+ // same value for both, only the older heap-profile collect separate
+ // stats for in-use and allocated objects.
+ valueIndex := 1
+ if LegacyHeapAllocated {
+ valueIndex = 3
+ }
+
+ var v1, v2 int64
+ if v1, err = strconv.ParseInt(sampleData[valueIndex], 10, 64); err != nil {
+ return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
+ }
+ if v2, err = strconv.ParseInt(sampleData[valueIndex+1], 10, 64); err != nil {
+ return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
+ }
+
+ if v1 == 0 {
+ if v2 != 0 {
+ return value, blocksize, addrs, fmt.Errorf("allocation count was 0 but allocation bytes was %d", v2)
+ }
+ } else {
+ blocksize = v2 / v1
+ if sampling == "v2" {
+ v1, v2 = scaleHeapSample(v1, v2, rate)
+ }
+ }
+
+ value = []int64{v1, v2}
+ addrs = parseHexAddresses(sampleData[5])
+
+ return value, blocksize, addrs, nil
+}
+
+// extractHexAddresses extracts hex numbers from a string and returns
+// them, together with their numeric value, in a slice.
+func extractHexAddresses(s string) ([]string, []uint64) {
+ hexStrings := hexNumberRE.FindAllString(s, -1)
+ var ids []uint64
+ for _, s := range hexStrings {
+ if id, err := strconv.ParseUint(s, 0, 64); err == nil {
+ ids = append(ids, id)
+ } else {
+ // Do not expect any parsing failures due to the regexp matching.
+ panic("failed to parse hex value:" + s)
+ }
+ }
+ return hexStrings, ids
+}
+
+// parseHexAddresses parses hex numbers from a string and returns them
+// in a slice.
+func parseHexAddresses(s string) []uint64 {
+ _, ids := extractHexAddresses(s)
+ return ids
+}
+
+// scaleHeapSample adjusts the data from a heapz Sample to
+// account for its probability of appearing in the collected
+// data. heapz profiles are a sampling of the memory allocations
+// requests in a program. We estimate the unsampled value by dividing
+// each collected sample by its probability of appearing in the
+// profile. heapz v2 profiles rely on a poisson process to determine
+// which samples to collect, based on the desired average collection
+// rate R. The probability of a sample of size S to appear in that
+// profile is 1-exp(-S/R).
+func scaleHeapSample(count, size, rate int64) (int64, int64) {
+ if count == 0 || size == 0 {
+ return 0, 0
+ }
+
+ if rate <= 1 {
+ // if rate==1 all samples were collected so no adjustment is needed.
+ // if rate<1 treat as unknown and skip scaling.
+ return count, size
+ }
+
+ avgSize := float64(size) / float64(count)
+ scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))
+
+ return int64(float64(count) * scale), int64(float64(size) * scale)
+}
+
+// parseContention parses a mutex or contention profile. There are 2 cases:
+// "--- contentionz " for legacy C++ profiles (and backwards compatibility)
+// "--- mutex:" or "--- contention:" for profiles generated by the Go runtime.
+// This code converts the text output from runtime into a *Profile. (In the future
+// the runtime might write a serialized Profile directly making this unnecessary.)
+func parseContention(b []byte) (*Profile, error) {
+ r := bytes.NewBuffer(b)
+ var l string
+ var err error
+ for {
+ // Skip past comments and empty lines seeking a real header.
+ l, err = r.ReadString('\n')
+ if err != nil {
+ return nil, err
+ }
+ if !isSpaceOrComment(l) {
+ break
+ }
+ }
+
+ if strings.HasPrefix(l, "--- contentionz ") {
+ return parseCppContention(r)
+ } else if strings.HasPrefix(l, "--- mutex:") {
+ return parseCppContention(r)
+ } else if strings.HasPrefix(l, "--- contention:") {
+ return parseCppContention(r)
+ }
+ return nil, errUnrecognized
+}
+
+// parseCppContention parses the output from synchronization_profiling.cc
+// for backward compatibility, and the compatible (non-debug) block profile
+// output from the Go runtime.
+func parseCppContention(r *bytes.Buffer) (*Profile, error) {
+ p := &Profile{
+ PeriodType: &ValueType{Type: "contentions", Unit: "count"},
+ Period: 1,
+ SampleType: []*ValueType{
+ {Type: "contentions", Unit: "count"},
+ {Type: "delay", Unit: "nanoseconds"},
+ },
+ }
+
+ var cpuHz int64
+ var l string
+ var err error
+ // Parse text of the form "attribute = value" before the samples.
+ const delimiter = '='
+ for {
+ l, err = r.ReadString('\n')
+ if err != nil {
+ if err != io.EOF {
+ return nil, err
+ }
+
+ if l == "" {
+ break
+ }
+ }
+ if isSpaceOrComment(l) {
+ continue
+ }
+
+ if l = strings.TrimSpace(l); l == "" {
+ continue
+ }
+
+ if strings.HasPrefix(l, "---") {
+ break
+ }
+
+ index := strings.IndexByte(l, delimiter)
+ if index < 0 {
+ break
+ }
+ key := l[:index]
+ val := l[index+1:]
+
+ key, val = strings.TrimSpace(key), strings.TrimSpace(val)
+ var err error
+ switch key {
+ case "cycles/second":
+ if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ case "sampling period":
+ if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ case "ms since reset":
+ ms, err := strconv.ParseInt(val, 0, 64)
+ if err != nil {
+ return nil, errUnrecognized
+ }
+ p.DurationNanos = ms * 1000 * 1000
+ case "format":
+ // CPP contentionz profiles don't have format.
+ return nil, errUnrecognized
+ case "resolution":
+ // CPP contentionz profiles don't have resolution.
+ return nil, errUnrecognized
+ case "discarded samples":
+ default:
+ return nil, errUnrecognized
+ }
+ }
+
+ locs := make(map[uint64]*Location)
+ for {
+ if !isSpaceOrComment(l) {
+ if l = strings.TrimSpace(l); strings.HasPrefix(l, "---") {
+ break
+ }
+ value, addrs, err := parseContentionSample(l, p.Period, cpuHz)
+ if err != nil {
+ return nil, err
+ }
+ var sloc []*Location
+ for _, addr := range addrs {
+ // Addresses from stack traces point to the next instruction after
+ // each call. Adjust by -1 to land somewhere on the actual call.
+ addr--
+ loc := locs[addr]
+ if locs[addr] == nil {
+ loc = &Location{
+ Address: addr,
+ }
+ p.Location = append(p.Location, loc)
+ locs[addr] = loc
+ }
+ sloc = append(sloc, loc)
+ }
+ p.Sample = append(p.Sample, &Sample{
+ Value: value,
+ Location: sloc,
+ })
+ }
+
+ if l, err = r.ReadString('\n'); err != nil {
+ if err != io.EOF {
+ return nil, err
+ }
+ if l == "" {
+ break
+ }
+ }
+ }
+
+ if err = parseAdditionalSections(l, r, p); err != nil {
+ return nil, err
+ }
+
+ return p, nil
+}
+
+// parseContentionSample parses a single row from a contention profile
+// into a new Sample.
+func parseContentionSample(line string, period, cpuHz int64) (value []int64, addrs []uint64, err error) {
+ sampleData := contentionSampleRE.FindStringSubmatch(line)
+ if sampleData == nil {
+ return value, addrs, errUnrecognized
+ }
+
+ v1, err := strconv.ParseInt(sampleData[1], 10, 64)
+ if err != nil {
+ return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
+ }
+ v2, err := strconv.ParseInt(sampleData[2], 10, 64)
+ if err != nil {
+ return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
+ }
+
+ // Unsample values if period and cpuHz are available.
+ // - Delays are scaled to cycles and then to nanoseconds.
+ // - Contentions are scaled to cycles.
+ if period > 0 {
+ if cpuHz > 0 {
+ cpuGHz := float64(cpuHz) / 1e9
+ v1 = int64(float64(v1) * float64(period) / cpuGHz)
+ }
+ v2 = v2 * period
+ }
+
+ value = []int64{v2, v1}
+ addrs = parseHexAddresses(sampleData[3])
+
+ return value, addrs, nil
+}
+
+// parseThread parses a Threadz profile and returns a new Profile.
+func parseThread(b []byte) (*Profile, error) {
+ r := bytes.NewBuffer(b)
+
+ var line string
+ var err error
+ for {
+ // Skip past comments and empty lines seeking a real header.
+ line, err = r.ReadString('\n')
+ if err != nil {
+ return nil, err
+ }
+ if !isSpaceOrComment(line) {
+ break
+ }
+ }
+
+ if m := threadzStartRE.FindStringSubmatch(line); m != nil {
+ // Advance over initial comments until first stack trace.
+ for {
+ line, err = r.ReadString('\n')
+ if err != nil {
+ if err != io.EOF {
+ return nil, err
+ }
+
+ if line == "" {
+ break
+ }
+ }
+ if sectionTrigger(line) != unrecognizedSection || line[0] == '-' {
+ break
+ }
+ }
+ } else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 {
+ return nil, errUnrecognized
+ }
+
+ p := &Profile{
+ SampleType: []*ValueType{{Type: "thread", Unit: "count"}},
+ PeriodType: &ValueType{Type: "thread", Unit: "count"},
+ Period: 1,
+ }
+
+ locs := make(map[uint64]*Location)
+ // Recognize each thread and populate profile samples.
+ for sectionTrigger(line) == unrecognizedSection {
+ if strings.HasPrefix(line, "---- no stack trace for") {
+ line = ""
+ break
+ }
+ if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 {
+ return nil, errUnrecognized
+ }
+
+ var addrs []uint64
+ line, addrs, err = parseThreadSample(r)
+ if err != nil {
+ return nil, errUnrecognized
+ }
+ if len(addrs) == 0 {
+ // We got a --same as previous threads--. Bump counters.
+ if len(p.Sample) > 0 {
+ s := p.Sample[len(p.Sample)-1]
+ s.Value[0]++
+ }
+ continue
+ }
+
+ var sloc []*Location
+ for _, addr := range addrs {
+ // Addresses from stack traces point to the next instruction after
+ // each call. Adjust by -1 to land somewhere on the actual call.
+ addr--
+ loc := locs[addr]
+ if locs[addr] == nil {
+ loc = &Location{
+ Address: addr,
+ }
+ p.Location = append(p.Location, loc)
+ locs[addr] = loc
+ }
+ sloc = append(sloc, loc)
+ }
+
+ p.Sample = append(p.Sample, &Sample{
+ Value: []int64{1},
+ Location: sloc,
+ })
+ }
+
+ if err = parseAdditionalSections(line, r, p); err != nil {
+ return nil, err
+ }
+
+ return p, nil
+}
+
+// parseThreadSample parses a symbolized or unsymbolized stack trace.
+// Returns the first line after the traceback, the sample (or nil if
+// it hits a 'same-as-previous' marker) and an error.
+func parseThreadSample(b *bytes.Buffer) (nextl string, addrs []uint64, err error) {
+ var l string
+ sameAsPrevious := false
+ for {
+ if l, err = b.ReadString('\n'); err != nil {
+ if err != io.EOF {
+ return "", nil, err
+ }
+ if l == "" {
+ break
+ }
+ }
+ if l = strings.TrimSpace(l); l == "" {
+ continue
+ }
+
+ if strings.HasPrefix(l, "---") {
+ break
+ }
+ if strings.Contains(l, "same as previous thread") {
+ sameAsPrevious = true
+ continue
+ }
+
+ addrs = append(addrs, parseHexAddresses(l)...)
+ }
+
+ if sameAsPrevious {
+ return l, nil, nil
+ }
+ return l, addrs, nil
+}
+
+// parseAdditionalSections parses any additional sections in the
+// profile, ignoring any unrecognized sections.
+func parseAdditionalSections(l string, b *bytes.Buffer, p *Profile) (err error) {
+ for {
+ if sectionTrigger(l) == memoryMapSection {
+ break
+ }
+ // Ignore any unrecognized sections.
+ if l, err := b.ReadString('\n'); err != nil {
+ if err != io.EOF {
+ return err
+ }
+ if l == "" {
+ break
+ }
+ }
+ }
+ return p.ParseMemoryMap(b)
+}
+
+// ParseMemoryMap parses a memory map in the format of
+// /proc/self/maps, and overrides the mappings in the current profile.
+// It renumbers the samples and locations in the profile correspondingly.
+func (p *Profile) ParseMemoryMap(rd io.Reader) error {
+ b := bufio.NewReader(rd)
+
+ var attrs []string
+ var r *strings.Replacer
+ const delimiter = '='
+ for {
+ l, err := b.ReadString('\n')
+ if err != nil {
+ if err != io.EOF {
+ return err
+ }
+ if l == "" {
+ break
+ }
+ }
+ if l = strings.TrimSpace(l); l == "" {
+ continue
+ }
+
+ if r != nil {
+ l = r.Replace(l)
+ }
+ m, err := parseMappingEntry(l)
+ if err != nil {
+ if err == errUnrecognized {
+ // Recognize assignments of the form: attr=value, and replace
+ // $attr with value on subsequent mappings.
+ idx := strings.IndexByte(l, delimiter)
+ if idx >= 0 {
+ attr := l[:idx]
+ value := l[idx+1:]
+ attrs = append(attrs, "$"+strings.TrimSpace(attr), strings.TrimSpace(value))
+ r = strings.NewReplacer(attrs...)
+ }
+ // Ignore any unrecognized entries
+ continue
+ }
+ return err
+ }
+ if m == nil || (m.File == "" && len(p.Mapping) != 0) {
+ // In some cases the first entry may include the address range
+ // but not the name of the file. It should be followed by
+ // another entry with the name.
+ continue
+ }
+ if len(p.Mapping) == 1 && p.Mapping[0].File == "" {
+ // Update the name if this is the entry following that empty one.
+ p.Mapping[0].File = m.File
+ continue
+ }
+ p.Mapping = append(p.Mapping, m)
+ }
+ p.remapLocationIDs()
+ p.remapFunctionIDs()
+ p.remapMappingIDs()
+ return nil
+}
+
+func parseMappingEntry(l string) (*Mapping, error) {
+ mapping := &Mapping{}
+ var err error
+ if me := procMapsRE.FindStringSubmatch(l); len(me) == 9 {
+ if !strings.Contains(me[3], "x") {
+ // Skip non-executable entries.
+ return nil, nil
+ }
+ if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ if me[4] != "" {
+ if mapping.Offset, err = strconv.ParseUint(me[4], 16, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ }
+ mapping.File = me[8]
+ return mapping, nil
+ }
+
+ if me := briefMapsRE.FindStringSubmatch(l); len(me) == 6 {
+ if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ mapping.File = me[3]
+ if me[5] != "" {
+ if mapping.Offset, err = strconv.ParseUint(me[5], 16, 64); err != nil {
+ return nil, errUnrecognized
+ }
+ }
+ return mapping, nil
+ }
+
+ return nil, errUnrecognized
+}
+
+type sectionType int
+
+const (
+ unrecognizedSection sectionType = iota
+ memoryMapSection
+)
+
+var memoryMapTriggers = []string{
+ "--- Memory map: ---",
+ "MAPPED_LIBRARIES:",
+}
+
+func sectionTrigger(line string) sectionType {
+ for _, trigger := range memoryMapTriggers {
+ if strings.Contains(line, trigger) {
+ return memoryMapSection
+ }
+ }
+ return unrecognizedSection
+}
+
+func (p *Profile) addLegacyFrameInfo() {
+ switch {
+ case isProfileType(p, heapzSampleTypes) ||
+ isProfileType(p, heapzInUseSampleTypes) ||
+ isProfileType(p, heapzAllocSampleTypes):
+ p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr
+ case isProfileType(p, contentionzSampleTypes):
+ p.DropFrames, p.KeepFrames = lockRxStr, ""
+ default:
+ p.DropFrames, p.KeepFrames = cpuProfilerRxStr, ""
+ }
+}
+
+var heapzSampleTypes = []string{"allocations", "size"} // early Go pprof profiles
+var heapzInUseSampleTypes = []string{"inuse_objects", "inuse_space"}
+var heapzAllocSampleTypes = []string{"alloc_objects", "alloc_space"}
+var contentionzSampleTypes = []string{"contentions", "delay"}
+
+func isProfileType(p *Profile, t []string) bool {
+ st := p.SampleType
+ if len(st) != len(t) {
+ return false
+ }
+
+ for i := range st {
+ if st[i].Type != t[i] {
+ return false
+ }
+ }
+ return true
+}
+
+var allocRxStr = strings.Join([]string{
+ // POSIX entry points.
+ `calloc`,
+ `cfree`,
+ `malloc`,
+ `free`,
+ `memalign`,
+ `do_memalign`,
+ `(__)?posix_memalign`,
+ `pvalloc`,
+ `valloc`,
+ `realloc`,
+
+ // TC malloc.
+ `tcmalloc::.*`,
+ `tc_calloc`,
+ `tc_cfree`,
+ `tc_malloc`,
+ `tc_free`,
+ `tc_memalign`,
+ `tc_posix_memalign`,
+ `tc_pvalloc`,
+ `tc_valloc`,
+ `tc_realloc`,
+ `tc_new`,
+ `tc_delete`,
+ `tc_newarray`,
+ `tc_deletearray`,
+ `tc_new_nothrow`,
+ `tc_newarray_nothrow`,
+
+ // Memory-allocation routines on OS X.
+ `malloc_zone_malloc`,
+ `malloc_zone_calloc`,
+ `malloc_zone_valloc`,
+ `malloc_zone_realloc`,
+ `malloc_zone_memalign`,
+ `malloc_zone_free`,
+
+ // Go runtime
+ `runtime\..*`,
+
+ // Other misc. memory allocation routines
+ `BaseArena::.*`,
+ `(::)?do_malloc_no_errno`,
+ `(::)?do_malloc_pages`,
+ `(::)?do_malloc`,
+ `DoSampledAllocation`,
+ `MallocedMemBlock::MallocedMemBlock`,
+ `_M_allocate`,
+ `__builtin_(vec_)?delete`,
+ `__builtin_(vec_)?new`,
+ `__gnu_cxx::new_allocator::allocate`,
+ `__libc_malloc`,
+ `__malloc_alloc_template::allocate`,
+ `allocate`,
+ `cpp_alloc`,
+ `operator new(\[\])?`,
+ `simple_alloc::allocate`,
+}, `|`)
+
+var allocSkipRxStr = strings.Join([]string{
+ // Preserve Go runtime frames that appear in the middle/bottom of
+ // the stack.
+ `runtime\.panic`,
+ `runtime\.reflectcall`,
+ `runtime\.call[0-9]*`,
+}, `|`)
+
+var cpuProfilerRxStr = strings.Join([]string{
+ `ProfileData::Add`,
+ `ProfileData::prof_handler`,
+ `CpuProfiler::prof_handler`,
+ `__pthread_sighandler`,
+ `__restore`,
+}, `|`)
+
+var lockRxStr = strings.Join([]string{
+ `RecordLockProfileData`,
+ `(base::)?RecordLockProfileData.*`,
+ `(base::)?SubmitMutexProfileData.*`,
+ `(base::)?SubmitSpinLockProfileData.*`,
+ `(Mutex::)?AwaitCommon.*`,
+ `(Mutex::)?Unlock.*`,
+ `(Mutex::)?UnlockSlow.*`,
+ `(Mutex::)?ReaderUnlock.*`,
+ `(MutexLock::)?~MutexLock.*`,
+ `(SpinLock::)?Unlock.*`,
+ `(SpinLock::)?SlowUnlock.*`,
+ `(SpinLockHolder::)?~SpinLockHolder.*`,
+}, `|`)
diff --git a/framework/rest/profile/merge.go b/framework/rest/profile/merge.go
new file mode 100644
index 00000000..3ea7d4cf
--- /dev/null
+++ b/framework/rest/profile/merge.go
@@ -0,0 +1,461 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package profile
+
+import (
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+)
+
+// Merge merges all the profiles in profs into a single Profile.
+// Returns a new profile independent of the input profiles. The merged
+// profile is compacted to eliminate unused samples, locations,
+// functions and mappings. Profiles must have identical profile sample
+// and period types or the merge will fail. profile.Period of the
+// resulting profile will be the maximum of all profiles, and
+// profile.TimeNanos will be the earliest nonzero one.
+func Merge(srcs []*Profile) (*Profile, error) {
+ if len(srcs) == 0 {
+ return nil, fmt.Errorf("no profiles to merge")
+ }
+ p, err := combineHeaders(srcs)
+ if err != nil {
+ return nil, err
+ }
+
+ pm := &profileMerger{
+ p: p,
+ samples: make(map[sampleKey]*Sample, len(srcs[0].Sample)),
+ locations: make(map[locationKey]*Location, len(srcs[0].Location)),
+ functions: make(map[functionKey]*Function, len(srcs[0].Function)),
+ mappings: make(map[mappingKey]*Mapping, len(srcs[0].Mapping)),
+ }
+
+ for _, src := range srcs {
+ // Clear the profile-specific hash tables
+ pm.locationsByID = make(map[uint64]*Location, len(src.Location))
+ pm.functionsByID = make(map[uint64]*Function, len(src.Function))
+ pm.mappingsByID = make(map[uint64]mapInfo, len(src.Mapping))
+
+ if len(pm.mappings) == 0 && len(src.Mapping) > 0 {
+ // The Mapping list has the property that the first mapping
+ // represents the main binary. Take the first Mapping we see,
+ // otherwise the operations below will add mappings in an
+ // arbitrary order.
+ pm.mapMapping(src.Mapping[0])
+ }
+
+ for _, s := range src.Sample {
+ if !isZeroSample(s) {
+ pm.mapSample(s)
+ }
+ }
+ }
+
+ for _, s := range p.Sample {
+ if isZeroSample(s) {
+ // If there are any zero samples, re-merge the profile to GC
+ // them.
+ return Merge([]*Profile{p})
+ }
+ }
+
+ return p, nil
+}
+
+// Normalize normalizes the source profile by multiplying each value in profile by the
+// ratio of the sum of the base profile's values of that sample type to the sum of the
+// source profile's value of that sample type.
+func (p *Profile) Normalize(pb *Profile) error {
+
+ if err := p.compatible(pb); err != nil {
+ return err
+ }
+
+ baseVals := make([]int64, len(p.SampleType))
+ for _, s := range pb.Sample {
+ for i, v := range s.Value {
+ baseVals[i] += v
+ }
+ }
+
+ srcVals := make([]int64, len(p.SampleType))
+ for _, s := range p.Sample {
+ for i, v := range s.Value {
+ srcVals[i] += v
+ }
+ }
+
+ normScale := make([]float64, len(baseVals))
+ for i := range baseVals {
+ if srcVals[i] == 0 {
+ normScale[i] = 0.0
+ } else {
+ normScale[i] = float64(baseVals[i]) / float64(srcVals[i])
+ }
+ }
+ p.ScaleN(normScale)
+ return nil
+}
+
+func isZeroSample(s *Sample) bool {
+ for _, v := range s.Value {
+ if v != 0 {
+ return false
+ }
+ }
+ return true
+}
+
+type profileMerger struct {
+ p *Profile
+
+ // Memoization tables within a profile.
+ locationsByID map[uint64]*Location
+ functionsByID map[uint64]*Function
+ mappingsByID map[uint64]mapInfo
+
+ // Memoization tables for profile entities.
+ samples map[sampleKey]*Sample
+ locations map[locationKey]*Location
+ functions map[functionKey]*Function
+ mappings map[mappingKey]*Mapping
+}
+
+type mapInfo struct {
+ m *Mapping
+ offset int64
+}
+
+func (pm *profileMerger) mapSample(src *Sample) *Sample {
+ s := &Sample{
+ Location: make([]*Location, len(src.Location)),
+ Value: make([]int64, len(src.Value)),
+ Label: make(map[string][]string, len(src.Label)),
+ NumLabel: make(map[string][]int64, len(src.NumLabel)),
+ NumUnit: make(map[string][]string, len(src.NumLabel)),
+ }
+ for i, l := range src.Location {
+ s.Location[i] = pm.mapLocation(l)
+ }
+ for k, v := range src.Label {
+ vv := make([]string, len(v))
+ copy(vv, v)
+ s.Label[k] = vv
+ }
+ for k, v := range src.NumLabel {
+ u := src.NumUnit[k]
+ vv := make([]int64, len(v))
+ uu := make([]string, len(u))
+ copy(vv, v)
+ copy(uu, u)
+ s.NumLabel[k] = vv
+ s.NumUnit[k] = uu
+ }
+ // Check memoization table. Must be done on the remapped location to
+ // account for the remapped mapping. Add current values to the
+ // existing sample.
+ k := s.key()
+ if ss, ok := pm.samples[k]; ok {
+ for i, v := range src.Value {
+ ss.Value[i] += v
+ }
+ return ss
+ }
+ copy(s.Value, src.Value)
+ pm.samples[k] = s
+ pm.p.Sample = append(pm.p.Sample, s)
+ return s
+}
+
+// key generates sampleKey to be used as a key for maps.
+func (sample *Sample) key() sampleKey {
+ ids := make([]string, len(sample.Location))
+ for i, l := range sample.Location {
+ ids[i] = strconv.FormatUint(l.ID, 16)
+ }
+
+ labels := make([]string, 0, len(sample.Label))
+ for k, v := range sample.Label {
+ labels = append(labels, fmt.Sprintf("%q%q", k, v))
+ }
+ sort.Strings(labels)
+
+ numlabels := make([]string, 0, len(sample.NumLabel))
+ for k, v := range sample.NumLabel {
+ numlabels = append(numlabels, fmt.Sprintf("%q%x%x", k, v, sample.NumUnit[k]))
+ }
+ sort.Strings(numlabels)
+
+ return sampleKey{
+ strings.Join(ids, "|"),
+ strings.Join(labels, ""),
+ strings.Join(numlabels, ""),
+ }
+}
+
+type sampleKey struct {
+ locations string
+ labels string
+ numlabels string
+}
+
+func (pm *profileMerger) mapLocation(src *Location) *Location {
+ if src == nil {
+ return nil
+ }
+
+ if l, ok := pm.locationsByID[src.ID]; ok {
+ pm.locationsByID[src.ID] = l
+ return l
+ }
+
+ mi := pm.mapMapping(src.Mapping)
+ l := &Location{
+ ID: uint64(len(pm.p.Location) + 1),
+ Mapping: mi.m,
+ Address: uint64(int64(src.Address) + mi.offset),
+ Line: make([]Line, len(src.Line)),
+ IsFolded: src.IsFolded,
+ }
+ for i, ln := range src.Line {
+ l.Line[i] = pm.mapLine(ln)
+ }
+ // Check memoization table. Must be done on the remapped location to
+ // account for the remapped mapping ID.
+ k := l.key()
+ if ll, ok := pm.locations[k]; ok {
+ pm.locationsByID[src.ID] = ll
+ return ll
+ }
+ pm.locationsByID[src.ID] = l
+ pm.locations[k] = l
+ pm.p.Location = append(pm.p.Location, l)
+ return l
+}
+
+// key generates locationKey to be used as a key for maps.
+func (l *Location) key() locationKey {
+ key := locationKey{
+ addr: l.Address,
+ isFolded: l.IsFolded,
+ }
+ if l.Mapping != nil {
+ // Normalizes address to handle address space randomization.
+ key.addr -= l.Mapping.Start
+ key.mappingID = l.Mapping.ID
+ }
+ lines := make([]string, len(l.Line)*2)
+ for i, line := range l.Line {
+ if line.Function != nil {
+ lines[i*2] = strconv.FormatUint(line.Function.ID, 16)
+ }
+ lines[i*2+1] = strconv.FormatInt(line.Line, 16)
+ }
+ key.lines = strings.Join(lines, "|")
+ return key
+}
+
+type locationKey struct {
+ addr, mappingID uint64
+ lines string
+ isFolded bool
+}
+
+func (pm *profileMerger) mapMapping(src *Mapping) mapInfo {
+ if src == nil {
+ return mapInfo{}
+ }
+
+ if mi, ok := pm.mappingsByID[src.ID]; ok {
+ return mi
+ }
+
+ // Check memoization tables.
+ mk := src.key()
+ if m, ok := pm.mappings[mk]; ok {
+ mi := mapInfo{m, int64(m.Start) - int64(src.Start)}
+ pm.mappingsByID[src.ID] = mi
+ return mi
+ }
+ m := &Mapping{
+ ID: uint64(len(pm.p.Mapping) + 1),
+ Start: src.Start,
+ Limit: src.Limit,
+ Offset: src.Offset,
+ File: src.File,
+ BuildID: src.BuildID,
+ HasFunctions: src.HasFunctions,
+ HasFilenames: src.HasFilenames,
+ HasLineNumbers: src.HasLineNumbers,
+ HasInlineFrames: src.HasInlineFrames,
+ }
+ pm.p.Mapping = append(pm.p.Mapping, m)
+
+ // Update memoization tables.
+ pm.mappings[mk] = m
+ mi := mapInfo{m, 0}
+ pm.mappingsByID[src.ID] = mi
+ return mi
+}
+
+// key generates encoded strings of Mapping to be used as a key for
+// maps.
+func (m *Mapping) key() mappingKey {
+ // Normalize addresses to handle address space randomization.
+ // Round up to next 4K boundary to avoid minor discrepancies.
+ const mapsizeRounding = 0x1000
+
+ size := m.Limit - m.Start
+ size = size + mapsizeRounding - 1
+ size = size - (size % mapsizeRounding)
+ key := mappingKey{
+ size: size,
+ offset: m.Offset,
+ }
+
+ switch {
+ case m.BuildID != "":
+ key.buildIDOrFile = m.BuildID
+ case m.File != "":
+ key.buildIDOrFile = m.File
+ default:
+ // A mapping containing neither build ID nor file name is a fake mapping. A
+ // key with empty buildIDOrFile is used for fake mappings so that they are
+ // treated as the same mapping during merging.
+ }
+ return key
+}
+
+type mappingKey struct {
+ size, offset uint64
+ buildIDOrFile string
+}
+
+func (pm *profileMerger) mapLine(src Line) Line {
+ ln := Line{
+ Function: pm.mapFunction(src.Function),
+ Line: src.Line,
+ }
+ return ln
+}
+
+func (pm *profileMerger) mapFunction(src *Function) *Function {
+ if src == nil {
+ return nil
+ }
+ if f, ok := pm.functionsByID[src.ID]; ok {
+ return f
+ }
+ k := src.key()
+ if f, ok := pm.functions[k]; ok {
+ pm.functionsByID[src.ID] = f
+ return f
+ }
+ f := &Function{
+ ID: uint64(len(pm.p.Function) + 1),
+ Name: src.Name,
+ SystemName: src.SystemName,
+ Filename: src.Filename,
+ StartLine: src.StartLine,
+ }
+ pm.functions[k] = f
+ pm.functionsByID[src.ID] = f
+ pm.p.Function = append(pm.p.Function, f)
+ return f
+}
+
+// key generates a struct to be used as a key for maps.
+func (f *Function) key() functionKey {
+ return functionKey{
+ f.StartLine,
+ f.Name,
+ f.SystemName,
+ f.Filename,
+ }
+}
+
+type functionKey struct {
+ startLine int64
+ name, systemName, fileName string
+}
+
+// combineHeaders checks that all profiles can be merged and returns
+// their combined profile.
+func combineHeaders(srcs []*Profile) (*Profile, error) {
+ for _, s := range srcs[1:] {
+ if err := srcs[0].compatible(s); err != nil {
+ return nil, err
+ }
+ }
+
+ var timeNanos, durationNanos, period int64
+ var comments []string
+ seenComments := map[string]bool{}
+ var defaultSampleType string
+ for _, s := range srcs {
+ if timeNanos == 0 || s.TimeNanos < timeNanos {
+ timeNanos = s.TimeNanos
+ }
+ durationNanos += s.DurationNanos
+ if period == 0 || period < s.Period {
+ period = s.Period
+ }
+ for _, c := range s.Comments {
+ if seen := seenComments[c]; !seen {
+ comments = append(comments, c)
+ seenComments[c] = true
+ }
+ }
+ if defaultSampleType == "" {
+ defaultSampleType = s.DefaultSampleType
+ }
+ }
+
+ p := &Profile{
+ SampleType: make([]*ValueType, len(srcs[0].SampleType)),
+
+ DropFrames: srcs[0].DropFrames,
+ KeepFrames: srcs[0].KeepFrames,
+
+ TimeNanos: timeNanos,
+ DurationNanos: durationNanos,
+ PeriodType: srcs[0].PeriodType,
+ Period: period,
+
+ Comments: comments,
+ DefaultSampleType: defaultSampleType,
+ }
+ copy(p.SampleType, srcs[0].SampleType)
+ return p, nil
+}
+
+// compatible determines if two profiles can be compared/merged.
+// returns nil if the profiles are compatible; otherwise an error with
+// details on the incompatibility.
+func (p *Profile) compatible(pb *Profile) error {
+ if !equalValueType(p.PeriodType, pb.PeriodType) {
+ return fmt.Errorf("incompatible period types %v and %v", p.PeriodType, pb.PeriodType)
+ }
+
+ if len(p.SampleType) != len(pb.SampleType) {
+ return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType)
+ }
+
+ for i := range p.SampleType {
+ if !equalValueType(p.SampleType[i], pb.SampleType[i]) {
+ return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType)
+ }
+ }
+ return nil
+}
+
+// equalValueType returns true if the two value types are semantically
+// equal. It ignores the internal fields used during encode/decode.
+func equalValueType(st1, st2 *ValueType) bool {
+ return st1.Type == st2.Type && st1.Unit == st2.Unit
+}
diff --git a/framework/rest/profile/profile.go b/framework/rest/profile/profile.go
new file mode 100644
index 00000000..d73e0052
--- /dev/null
+++ b/framework/rest/profile/profile.go
@@ -0,0 +1,622 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package profile provides a representation of
+// github.com/google/pprof/proto/profile.proto and
+// methods to encode/decode/merge profiles in this format.
+package profile
+
+import (
+ "bytes"
+ "compress/gzip"
+ "fmt"
+ "github.com/unionj-cloud/go-doudou/v2/framework/rest/lazyregexp"
+ "io"
+ "strings"
+ "time"
+)
+
+// Profile is an in-memory representation of profile.proto.
+type Profile struct {
+ SampleType []*ValueType
+ DefaultSampleType string
+ Sample []*Sample
+ Mapping []*Mapping
+ Location []*Location
+ Function []*Function
+ Comments []string
+
+ DropFrames string
+ KeepFrames string
+
+ TimeNanos int64
+ DurationNanos int64
+ PeriodType *ValueType
+ Period int64
+
+ commentX []int64
+ dropFramesX int64
+ keepFramesX int64
+ stringTable []string
+ defaultSampleTypeX int64
+}
+
+// ValueType corresponds to Profile.ValueType
+type ValueType struct {
+ Type string // cpu, wall, inuse_space, etc
+ Unit string // seconds, nanoseconds, bytes, etc
+
+ typeX int64
+ unitX int64
+}
+
+// Sample corresponds to Profile.Sample
+type Sample struct {
+ Location []*Location
+ Value []int64
+ Label map[string][]string
+ NumLabel map[string][]int64
+ NumUnit map[string][]string
+
+ locationIDX []uint64
+ labelX []Label
+}
+
+// Label corresponds to Profile.Label
+type Label struct {
+ keyX int64
+ // Exactly one of the two following values must be set
+ strX int64
+ numX int64 // Integer value for this label
+}
+
+// Mapping corresponds to Profile.Mapping
+type Mapping struct {
+ ID uint64
+ Start uint64
+ Limit uint64
+ Offset uint64
+ File string
+ BuildID string
+ HasFunctions bool
+ HasFilenames bool
+ HasLineNumbers bool
+ HasInlineFrames bool
+
+ fileX int64
+ buildIDX int64
+}
+
+// Location corresponds to Profile.Location
+type Location struct {
+ ID uint64
+ Mapping *Mapping
+ Address uint64
+ Line []Line
+ IsFolded bool
+
+ mappingIDX uint64
+}
+
+// Line corresponds to Profile.Line
+type Line struct {
+ Function *Function
+ Line int64
+
+ functionIDX uint64
+}
+
+// Function corresponds to Profile.Function
+type Function struct {
+ ID uint64
+ Name string
+ SystemName string
+ Filename string
+ StartLine int64
+
+ nameX int64
+ systemNameX int64
+ filenameX int64
+}
+
+// Parse parses a profile and checks for its validity. The input
+// may be a gzip-compressed encoded protobuf or one of many legacy
+// profile formats which may be unsupported in the future.
+func Parse(r io.Reader) (*Profile, error) {
+ orig, err := io.ReadAll(r)
+ if err != nil {
+ return nil, err
+ }
+
+ var p *Profile
+ if len(orig) >= 2 && orig[0] == 0x1f && orig[1] == 0x8b {
+ gz, err := gzip.NewReader(bytes.NewBuffer(orig))
+ if err != nil {
+ return nil, fmt.Errorf("decompressing profile: %v", err)
+ }
+ data, err := io.ReadAll(gz)
+ if err != nil {
+ return nil, fmt.Errorf("decompressing profile: %v", err)
+ }
+ orig = data
+ }
+
+ var lErr error
+ p, pErr := parseUncompressed(orig)
+ if pErr != nil {
+ p, lErr = parseLegacy(orig)
+ }
+ if pErr != nil && lErr != nil {
+ return nil, fmt.Errorf("parsing profile: not a valid proto profile (%w) or legacy profile (%w)", pErr, lErr)
+ }
+
+ if err := p.CheckValid(); err != nil {
+ return nil, fmt.Errorf("malformed profile: %v", err)
+ }
+ return p, nil
+}
+
+var errUnrecognized = fmt.Errorf("unrecognized profile format")
+var errMalformed = fmt.Errorf("malformed profile format")
+var ErrNoData = fmt.Errorf("empty input file")
+
+func parseLegacy(data []byte) (*Profile, error) {
+ parsers := []func([]byte) (*Profile, error){
+ parseCPU,
+ parseHeap,
+ parseGoCount, // goroutine, threadcreate
+ parseThread,
+ parseContention,
+ }
+
+ for _, parser := range parsers {
+ p, err := parser(data)
+ if err == nil {
+ p.setMain()
+ p.addLegacyFrameInfo()
+ return p, nil
+ }
+ if err != errUnrecognized {
+ return nil, err
+ }
+ }
+ return nil, errUnrecognized
+}
+
+func parseUncompressed(data []byte) (*Profile, error) {
+ if len(data) == 0 {
+ return nil, ErrNoData
+ }
+
+ p := &Profile{}
+ if err := unmarshal(data, p); err != nil {
+ return nil, err
+ }
+
+ if err := p.postDecode(); err != nil {
+ return nil, err
+ }
+
+ return p, nil
+}
+
+var libRx = lazyregexp.New(`([.]so$|[.]so[._][0-9]+)`)
+
+// setMain scans Mapping entries and guesses which entry is main
+// because legacy profiles don't obey the convention of putting main
+// first.
+func (p *Profile) setMain() {
+ for i := 0; i < len(p.Mapping); i++ {
+ file := strings.TrimSpace(strings.ReplaceAll(p.Mapping[i].File, "(deleted)", ""))
+ if len(file) == 0 {
+ continue
+ }
+ if len(libRx.FindStringSubmatch(file)) > 0 {
+ continue
+ }
+ if strings.HasPrefix(file, "[") {
+ continue
+ }
+ // Swap what we guess is main to position 0.
+ p.Mapping[i], p.Mapping[0] = p.Mapping[0], p.Mapping[i]
+ break
+ }
+}
+
+// Write writes the profile as a gzip-compressed marshaled protobuf.
+func (p *Profile) Write(w io.Writer) error {
+ p.preEncode()
+ b := marshal(p)
+ zw := gzip.NewWriter(w)
+ defer zw.Close()
+ _, err := zw.Write(b)
+ return err
+}
+
+// CheckValid tests whether the profile is valid. Checks include, but are
+// not limited to:
+// - len(Profile.Sample[n].value) == len(Profile.value_unit)
+// - Sample.id has a corresponding Profile.Location
+func (p *Profile) CheckValid() error {
+ // Check that sample values are consistent
+ sampleLen := len(p.SampleType)
+ if sampleLen == 0 && len(p.Sample) != 0 {
+ return fmt.Errorf("missing sample type information")
+ }
+ for _, s := range p.Sample {
+ if len(s.Value) != sampleLen {
+ return fmt.Errorf("mismatch: sample has: %d values vs. %d types", len(s.Value), len(p.SampleType))
+ }
+ }
+
+ // Check that all mappings/locations/functions are in the tables
+ // Check that there are no duplicate ids
+ mappings := make(map[uint64]*Mapping, len(p.Mapping))
+ for _, m := range p.Mapping {
+ if m.ID == 0 {
+ return fmt.Errorf("found mapping with reserved ID=0")
+ }
+ if mappings[m.ID] != nil {
+ return fmt.Errorf("multiple mappings with same id: %d", m.ID)
+ }
+ mappings[m.ID] = m
+ }
+ functions := make(map[uint64]*Function, len(p.Function))
+ for _, f := range p.Function {
+ if f.ID == 0 {
+ return fmt.Errorf("found function with reserved ID=0")
+ }
+ if functions[f.ID] != nil {
+ return fmt.Errorf("multiple functions with same id: %d", f.ID)
+ }
+ functions[f.ID] = f
+ }
+ locations := make(map[uint64]*Location, len(p.Location))
+ for _, l := range p.Location {
+ if l.ID == 0 {
+ return fmt.Errorf("found location with reserved id=0")
+ }
+ if locations[l.ID] != nil {
+ return fmt.Errorf("multiple locations with same id: %d", l.ID)
+ }
+ locations[l.ID] = l
+ if m := l.Mapping; m != nil {
+ if m.ID == 0 || mappings[m.ID] != m {
+ return fmt.Errorf("inconsistent mapping %p: %d", m, m.ID)
+ }
+ }
+ for _, ln := range l.Line {
+ if f := ln.Function; f != nil {
+ if f.ID == 0 || functions[f.ID] != f {
+ return fmt.Errorf("inconsistent function %p: %d", f, f.ID)
+ }
+ }
+ }
+ }
+ return nil
+}
+
+// Aggregate merges the locations in the profile into equivalence
+// classes preserving the request attributes. It also updates the
+// samples to point to the merged locations.
+func (p *Profile) Aggregate(inlineFrame, function, filename, linenumber, address bool) error {
+ for _, m := range p.Mapping {
+ m.HasInlineFrames = m.HasInlineFrames && inlineFrame
+ m.HasFunctions = m.HasFunctions && function
+ m.HasFilenames = m.HasFilenames && filename
+ m.HasLineNumbers = m.HasLineNumbers && linenumber
+ }
+
+ // Aggregate functions
+ if !function || !filename {
+ for _, f := range p.Function {
+ if !function {
+ f.Name = ""
+ f.SystemName = ""
+ }
+ if !filename {
+ f.Filename = ""
+ }
+ }
+ }
+
+ // Aggregate locations
+ if !inlineFrame || !address || !linenumber {
+ for _, l := range p.Location {
+ if !inlineFrame && len(l.Line) > 1 {
+ l.Line = l.Line[len(l.Line)-1:]
+ }
+ if !linenumber {
+ for i := range l.Line {
+ l.Line[i].Line = 0
+ }
+ }
+ if !address {
+ l.Address = 0
+ }
+ }
+ }
+
+ return p.CheckValid()
+}
+
+// Print dumps a text representation of a profile. Intended mainly
+// for debugging purposes.
+func (p *Profile) String() string {
+
+ ss := make([]string, 0, len(p.Sample)+len(p.Mapping)+len(p.Location))
+ if pt := p.PeriodType; pt != nil {
+ ss = append(ss, fmt.Sprintf("PeriodType: %s %s", pt.Type, pt.Unit))
+ }
+ ss = append(ss, fmt.Sprintf("Period: %d", p.Period))
+ if p.TimeNanos != 0 {
+ ss = append(ss, fmt.Sprintf("Time: %v", time.Unix(0, p.TimeNanos)))
+ }
+ if p.DurationNanos != 0 {
+ ss = append(ss, fmt.Sprintf("Duration: %v", time.Duration(p.DurationNanos)))
+ }
+
+ ss = append(ss, "Samples:")
+ var sh1 string
+ for _, s := range p.SampleType {
+ sh1 = sh1 + fmt.Sprintf("%s/%s ", s.Type, s.Unit)
+ }
+ ss = append(ss, strings.TrimSpace(sh1))
+ for _, s := range p.Sample {
+ var sv string
+ for _, v := range s.Value {
+ sv = fmt.Sprintf("%s %10d", sv, v)
+ }
+ sv = sv + ": "
+ for _, l := range s.Location {
+ sv = sv + fmt.Sprintf("%d ", l.ID)
+ }
+ ss = append(ss, sv)
+ const labelHeader = " "
+ if len(s.Label) > 0 {
+ ls := labelHeader
+ for k, v := range s.Label {
+ ls = ls + fmt.Sprintf("%s:%v ", k, v)
+ }
+ ss = append(ss, ls)
+ }
+ if len(s.NumLabel) > 0 {
+ ls := labelHeader
+ for k, v := range s.NumLabel {
+ ls = ls + fmt.Sprintf("%s:%v ", k, v)
+ }
+ ss = append(ss, ls)
+ }
+ }
+
+ ss = append(ss, "Locations")
+ for _, l := range p.Location {
+ locStr := fmt.Sprintf("%6d: %#x ", l.ID, l.Address)
+ if m := l.Mapping; m != nil {
+ locStr = locStr + fmt.Sprintf("M=%d ", m.ID)
+ }
+ if len(l.Line) == 0 {
+ ss = append(ss, locStr)
+ }
+ for li := range l.Line {
+ lnStr := "??"
+ if fn := l.Line[li].Function; fn != nil {
+ lnStr = fmt.Sprintf("%s %s:%d s=%d",
+ fn.Name,
+ fn.Filename,
+ l.Line[li].Line,
+ fn.StartLine)
+ if fn.Name != fn.SystemName {
+ lnStr = lnStr + "(" + fn.SystemName + ")"
+ }
+ }
+ ss = append(ss, locStr+lnStr)
+ // Do not print location details past the first line
+ locStr = " "
+ }
+ }
+
+ ss = append(ss, "Mappings")
+ for _, m := range p.Mapping {
+ bits := ""
+ if m.HasFunctions {
+ bits += "[FN]"
+ }
+ if m.HasFilenames {
+ bits += "[FL]"
+ }
+ if m.HasLineNumbers {
+ bits += "[LN]"
+ }
+ if m.HasInlineFrames {
+ bits += "[IN]"
+ }
+ ss = append(ss, fmt.Sprintf("%d: %#x/%#x/%#x %s %s %s",
+ m.ID,
+ m.Start, m.Limit, m.Offset,
+ m.File,
+ m.BuildID,
+ bits))
+ }
+
+ return strings.Join(ss, "\n") + "\n"
+}
+
+// Merge adds profile p adjusted by ratio r into profile p. Profiles
+// must be compatible (same Type and SampleType).
+// TODO(rsilvera): consider normalizing the profiles based on the
+// total samples collected.
+func (p *Profile) Merge(pb *Profile, r float64) error {
+ if err := p.Compatible(pb); err != nil {
+ return err
+ }
+
+ pb = pb.Copy()
+
+ // Keep the largest of the two periods.
+ if pb.Period > p.Period {
+ p.Period = pb.Period
+ }
+
+ p.DurationNanos += pb.DurationNanos
+
+ p.Mapping = append(p.Mapping, pb.Mapping...)
+ for i, m := range p.Mapping {
+ m.ID = uint64(i + 1)
+ }
+ p.Location = append(p.Location, pb.Location...)
+ for i, l := range p.Location {
+ l.ID = uint64(i + 1)
+ }
+ p.Function = append(p.Function, pb.Function...)
+ for i, f := range p.Function {
+ f.ID = uint64(i + 1)
+ }
+
+ if r != 1.0 {
+ for _, s := range pb.Sample {
+ for i, v := range s.Value {
+ s.Value[i] = int64((float64(v) * r))
+ }
+ }
+ }
+ p.Sample = append(p.Sample, pb.Sample...)
+ return p.CheckValid()
+}
+
+// Compatible determines if two profiles can be compared/merged.
+// returns nil if the profiles are compatible; otherwise an error with
+// details on the incompatibility.
+func (p *Profile) Compatible(pb *Profile) error {
+ if !compatibleValueTypes(p.PeriodType, pb.PeriodType) {
+ return fmt.Errorf("incompatible period types %v and %v", p.PeriodType, pb.PeriodType)
+ }
+
+ if len(p.SampleType) != len(pb.SampleType) {
+ return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType)
+ }
+
+ for i := range p.SampleType {
+ if !compatibleValueTypes(p.SampleType[i], pb.SampleType[i]) {
+ return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType)
+ }
+ }
+
+ return nil
+}
+
+// HasFunctions determines if all locations in this profile have
+// symbolized function information.
+func (p *Profile) HasFunctions() bool {
+ for _, l := range p.Location {
+ if l.Mapping == nil || !l.Mapping.HasFunctions {
+ return false
+ }
+ }
+ return true
+}
+
+// HasFileLines determines if all locations in this profile have
+// symbolized file and line number information.
+func (p *Profile) HasFileLines() bool {
+ for _, l := range p.Location {
+ if l.Mapping == nil || (!l.Mapping.HasFilenames || !l.Mapping.HasLineNumbers) {
+ return false
+ }
+ }
+ return true
+}
+
+func compatibleValueTypes(v1, v2 *ValueType) bool {
+ if v1 == nil || v2 == nil {
+ return true // No grounds to disqualify.
+ }
+ return v1.Type == v2.Type && v1.Unit == v2.Unit
+}
+
+// Copy makes a fully independent copy of a profile.
+func (p *Profile) Copy() *Profile {
+ p.preEncode()
+ b := marshal(p)
+
+ pp := &Profile{}
+ if err := unmarshal(b, pp); err != nil {
+ panic(err)
+ }
+ if err := pp.postDecode(); err != nil {
+ panic(err)
+ }
+
+ return pp
+}
+
+// Demangler maps symbol names to a human-readable form. This may
+// include C++ demangling and additional simplification. Names that
+// are not demangled may be missing from the resulting map.
+type Demangler func(name []string) (map[string]string, error)
+
+// Demangle attempts to demangle and optionally simplify any function
+// names referenced in the profile. It works on a best-effort basis:
+// it will silently preserve the original names in case of any errors.
+func (p *Profile) Demangle(d Demangler) error {
+ // Collect names to demangle.
+ var names []string
+ for _, fn := range p.Function {
+ names = append(names, fn.SystemName)
+ }
+
+ // Update profile with demangled names.
+ demangled, err := d(names)
+ if err != nil {
+ return err
+ }
+ for _, fn := range p.Function {
+ if dd, ok := demangled[fn.SystemName]; ok {
+ fn.Name = dd
+ }
+ }
+ return nil
+}
+
+// Empty reports whether the profile contains no samples.
+func (p *Profile) Empty() bool {
+ return len(p.Sample) == 0
+}
+
+// Scale multiplies all sample values in a profile by a constant.
+func (p *Profile) Scale(ratio float64) {
+ if ratio == 1 {
+ return
+ }
+ ratios := make([]float64, len(p.SampleType))
+ for i := range p.SampleType {
+ ratios[i] = ratio
+ }
+ p.ScaleN(ratios)
+}
+
+// ScaleN multiplies each sample values in a sample by a different amount.
+func (p *Profile) ScaleN(ratios []float64) error {
+ if len(p.SampleType) != len(ratios) {
+ return fmt.Errorf("mismatched scale ratios, got %d, want %d", len(ratios), len(p.SampleType))
+ }
+ allOnes := true
+ for _, r := range ratios {
+ if r != 1 {
+ allOnes = false
+ break
+ }
+ }
+ if allOnes {
+ return nil
+ }
+ for _, s := range p.Sample {
+ for i, v := range s.Value {
+ if ratios[i] != 1 {
+ s.Value[i] = int64(float64(v) * ratios[i])
+ }
+ }
+ }
+ return nil
+}
diff --git a/framework/rest/profile/profile_test.go b/framework/rest/profile/profile_test.go
new file mode 100644
index 00000000..84158b62
--- /dev/null
+++ b/framework/rest/profile/profile_test.go
@@ -0,0 +1,64 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package profile
+
+import (
+ "testing"
+)
+
+func TestParseContention(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ wantErr bool
+ }{
+ {
+ name: "valid",
+ in: `--- mutex:
+cycles/second=3491920901
+sampling period=1
+43227965305 1659640 @ 0x45e851 0x45f764 0x4a2be1 0x44ea31
+34035731690 15760 @ 0x45e851 0x45f764 0x4a2b17 0x44ea31
+`,
+ },
+ {
+ name: "valid with comment",
+ in: `--- mutex:
+cycles/second=3491920901
+sampling period=1
+43227965305 1659640 @ 0x45e851 0x45f764 0x4a2be1 0x44ea31
+# 0x45e850 sync.(*Mutex).Unlock+0x80 /go/src/sync/mutex.go:126
+# 0x45f763 sync.(*RWMutex).Unlock+0x83 /go/src/sync/rwmutex.go:125
+# 0x4a2be0 main.main.func3+0x70 /go/src/internal/pprof/profile/a_binary.go:58
+
+34035731690 15760 @ 0x45e851 0x45f764 0x4a2b17 0x44ea31
+# 0x45e850 sync.(*Mutex).Unlock+0x80 /go/src/sync/mutex.go:126
+# 0x45f763 sync.(*RWMutex).Unlock+0x83 /go/src/sync/rwmutex.go:125
+# 0x4a2b16 main.main.func2+0xd6 /go/src/internal/pprof/profile/a_binary.go:48
+`,
+ },
+ {
+ name: "empty",
+ in: `--- mutex:`,
+ wantErr: true,
+ },
+ {
+ name: "invalid header",
+ in: `--- channel:
+43227965305 1659640 @ 0x45e851 0x45f764 0x4a2be1 0x44ea31`,
+ wantErr: true,
+ },
+ }
+ for _, tc := range tests {
+ _, err := parseContention([]byte(tc.in))
+ if tc.wantErr && err == nil {
+ t.Errorf("parseContention(%q) succeeded unexpectedly", tc.name)
+ }
+ if !tc.wantErr && err != nil {
+ t.Errorf("parseContention(%q) failed unexpectedly: %v", tc.name, err)
+ }
+ }
+
+}
diff --git a/framework/rest/profile/proto.go b/framework/rest/profile/proto.go
new file mode 100644
index 00000000..58ff0ad2
--- /dev/null
+++ b/framework/rest/profile/proto.go
@@ -0,0 +1,356 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is a simple protocol buffer encoder and decoder.
+//
+// A protocol message must implement the message interface:
+// decoder() []decoder
+// encode(*buffer)
+//
+// The decode method returns a slice indexed by field number that gives the
+// function to decode that field.
+// The encode method encodes its receiver into the given buffer.
+//
+// The two methods are simple enough to be implemented by hand rather than
+// by using a protocol compiler.
+//
+// See profile.go for examples of messages implementing this interface.
+//
+// There is no support for groups, message sets, or "has" bits.
+
+package profile
+
+import (
+ "errors"
+ "fmt"
+)
+
+type buffer struct {
+ field int
+ typ int
+ u64 uint64
+ data []byte
+ tmp [16]byte
+}
+
+type decoder func(*buffer, message) error
+
+type message interface {
+ decoder() []decoder
+ encode(*buffer)
+}
+
+func marshal(m message) []byte {
+ var b buffer
+ m.encode(&b)
+ return b.data
+}
+
+func encodeVarint(b *buffer, x uint64) {
+ for x >= 128 {
+ b.data = append(b.data, byte(x)|0x80)
+ x >>= 7
+ }
+ b.data = append(b.data, byte(x))
+}
+
+func encodeLength(b *buffer, tag int, len int) {
+ encodeVarint(b, uint64(tag)<<3|2)
+ encodeVarint(b, uint64(len))
+}
+
+func encodeUint64(b *buffer, tag int, x uint64) {
+ // append varint to b.data
+ encodeVarint(b, uint64(tag)<<3|0)
+ encodeVarint(b, x)
+}
+
+func encodeUint64s(b *buffer, tag int, x []uint64) {
+ if len(x) > 2 {
+ // Use packed encoding
+ n1 := len(b.data)
+ for _, u := range x {
+ encodeVarint(b, u)
+ }
+ n2 := len(b.data)
+ encodeLength(b, tag, n2-n1)
+ n3 := len(b.data)
+ copy(b.tmp[:], b.data[n2:n3])
+ copy(b.data[n1+(n3-n2):], b.data[n1:n2])
+ copy(b.data[n1:], b.tmp[:n3-n2])
+ return
+ }
+ for _, u := range x {
+ encodeUint64(b, tag, u)
+ }
+}
+
+func encodeUint64Opt(b *buffer, tag int, x uint64) {
+ if x == 0 {
+ return
+ }
+ encodeUint64(b, tag, x)
+}
+
+func encodeInt64(b *buffer, tag int, x int64) {
+ u := uint64(x)
+ encodeUint64(b, tag, u)
+}
+
+func encodeInt64Opt(b *buffer, tag int, x int64) {
+ if x == 0 {
+ return
+ }
+ encodeInt64(b, tag, x)
+}
+
+func encodeInt64s(b *buffer, tag int, x []int64) {
+ if len(x) > 2 {
+ // Use packed encoding
+ n1 := len(b.data)
+ for _, u := range x {
+ encodeVarint(b, uint64(u))
+ }
+ n2 := len(b.data)
+ encodeLength(b, tag, n2-n1)
+ n3 := len(b.data)
+ copy(b.tmp[:], b.data[n2:n3])
+ copy(b.data[n1+(n3-n2):], b.data[n1:n2])
+ copy(b.data[n1:], b.tmp[:n3-n2])
+ return
+ }
+ for _, u := range x {
+ encodeInt64(b, tag, u)
+ }
+}
+
+func encodeString(b *buffer, tag int, x string) {
+ encodeLength(b, tag, len(x))
+ b.data = append(b.data, x...)
+}
+
+func encodeStrings(b *buffer, tag int, x []string) {
+ for _, s := range x {
+ encodeString(b, tag, s)
+ }
+}
+
+func encodeBool(b *buffer, tag int, x bool) {
+ if x {
+ encodeUint64(b, tag, 1)
+ } else {
+ encodeUint64(b, tag, 0)
+ }
+}
+
+func encodeBoolOpt(b *buffer, tag int, x bool) {
+ if !x {
+ return
+ }
+ encodeBool(b, tag, x)
+}
+
+func encodeMessage(b *buffer, tag int, m message) {
+ n1 := len(b.data)
+ m.encode(b)
+ n2 := len(b.data)
+ encodeLength(b, tag, n2-n1)
+ n3 := len(b.data)
+ copy(b.tmp[:], b.data[n2:n3])
+ copy(b.data[n1+(n3-n2):], b.data[n1:n2])
+ copy(b.data[n1:], b.tmp[:n3-n2])
+}
+
+func unmarshal(data []byte, m message) (err error) {
+ b := buffer{data: data, typ: 2}
+ return decodeMessage(&b, m)
+}
+
+func le64(p []byte) uint64 {
+ return uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56
+}
+
+func le32(p []byte) uint32 {
+ return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
+}
+
+func decodeVarint(data []byte) (uint64, []byte, error) {
+ var i int
+ var u uint64
+ for i = 0; ; i++ {
+ if i >= 10 || i >= len(data) {
+ return 0, nil, errors.New("bad varint")
+ }
+ u |= uint64(data[i]&0x7F) << uint(7*i)
+ if data[i]&0x80 == 0 {
+ return u, data[i+1:], nil
+ }
+ }
+}
+
+func decodeField(b *buffer, data []byte) ([]byte, error) {
+ x, data, err := decodeVarint(data)
+ if err != nil {
+ return nil, err
+ }
+ b.field = int(x >> 3)
+ b.typ = int(x & 7)
+ b.data = nil
+ b.u64 = 0
+ switch b.typ {
+ case 0:
+ b.u64, data, err = decodeVarint(data)
+ if err != nil {
+ return nil, err
+ }
+ case 1:
+ if len(data) < 8 {
+ return nil, errors.New("not enough data")
+ }
+ b.u64 = le64(data[:8])
+ data = data[8:]
+ case 2:
+ var n uint64
+ n, data, err = decodeVarint(data)
+ if err != nil {
+ return nil, err
+ }
+ if n > uint64(len(data)) {
+ return nil, errors.New("too much data")
+ }
+ b.data = data[:n]
+ data = data[n:]
+ case 5:
+ if len(data) < 4 {
+ return nil, errors.New("not enough data")
+ }
+ b.u64 = uint64(le32(data[:4]))
+ data = data[4:]
+ default:
+ return nil, fmt.Errorf("unknown wire type: %d", b.typ)
+ }
+
+ return data, nil
+}
+
+func checkType(b *buffer, typ int) error {
+ if b.typ != typ {
+ return errors.New("type mismatch")
+ }
+ return nil
+}
+
+func decodeMessage(b *buffer, m message) error {
+ if err := checkType(b, 2); err != nil {
+ return err
+ }
+ dec := m.decoder()
+ data := b.data
+ for len(data) > 0 {
+ // pull varint field# + type
+ var err error
+ data, err = decodeField(b, data)
+ if err != nil {
+ return err
+ }
+ if b.field >= len(dec) || dec[b.field] == nil {
+ continue
+ }
+ if err := dec[b.field](b, m); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func decodeInt64(b *buffer, x *int64) error {
+ if err := checkType(b, 0); err != nil {
+ return err
+ }
+ *x = int64(b.u64)
+ return nil
+}
+
+func decodeInt64s(b *buffer, x *[]int64) error {
+ if b.typ == 2 {
+ // Packed encoding
+ data := b.data
+ for len(data) > 0 {
+ var u uint64
+ var err error
+
+ if u, data, err = decodeVarint(data); err != nil {
+ return err
+ }
+ *x = append(*x, int64(u))
+ }
+ return nil
+ }
+ var i int64
+ if err := decodeInt64(b, &i); err != nil {
+ return err
+ }
+ *x = append(*x, i)
+ return nil
+}
+
+func decodeUint64(b *buffer, x *uint64) error {
+ if err := checkType(b, 0); err != nil {
+ return err
+ }
+ *x = b.u64
+ return nil
+}
+
+func decodeUint64s(b *buffer, x *[]uint64) error {
+ if b.typ == 2 {
+ data := b.data
+ // Packed encoding
+ for len(data) > 0 {
+ var u uint64
+ var err error
+
+ if u, data, err = decodeVarint(data); err != nil {
+ return err
+ }
+ *x = append(*x, u)
+ }
+ return nil
+ }
+ var u uint64
+ if err := decodeUint64(b, &u); err != nil {
+ return err
+ }
+ *x = append(*x, u)
+ return nil
+}
+
+func decodeString(b *buffer, x *string) error {
+ if err := checkType(b, 2); err != nil {
+ return err
+ }
+ *x = string(b.data)
+ return nil
+}
+
+func decodeStrings(b *buffer, x *[]string) error {
+ var s string
+ if err := decodeString(b, &s); err != nil {
+ return err
+ }
+ *x = append(*x, s)
+ return nil
+}
+
+func decodeBool(b *buffer, x *bool) error {
+ if err := checkType(b, 0); err != nil {
+ return err
+ }
+ if int64(b.u64) == 0 {
+ *x = false
+ } else {
+ *x = true
+ }
+ return nil
+}
diff --git a/framework/rest/profile/proto_test.go b/framework/rest/profile/proto_test.go
new file mode 100644
index 00000000..46c6d830
--- /dev/null
+++ b/framework/rest/profile/proto_test.go
@@ -0,0 +1,71 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package profile
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestPackedEncoding(t *testing.T) {
+
+ type testcase struct {
+ uint64s []uint64
+ int64s []int64
+ encoded []byte
+ }
+ for i, tc := range []testcase{
+ {
+ []uint64{0, 1, 10, 100, 1000, 10000},
+ []int64{1000, 0, 1000},
+ []byte{10, 8, 0, 1, 10, 100, 232, 7, 144, 78, 18, 5, 232, 7, 0, 232, 7},
+ },
+ {
+ []uint64{10000},
+ nil,
+ []byte{8, 144, 78},
+ },
+ {
+ nil,
+ []int64{-10000},
+ []byte{16, 240, 177, 255, 255, 255, 255, 255, 255, 255, 1},
+ },
+ } {
+ source := &packedInts{tc.uint64s, tc.int64s}
+ if got, want := marshal(source), tc.encoded; !reflect.DeepEqual(got, want) {
+ t.Errorf("failed encode %d, got %v, want %v", i, got, want)
+ }
+
+ dest := new(packedInts)
+ if err := unmarshal(tc.encoded, dest); err != nil {
+ t.Errorf("failed decode %d: %v", i, err)
+ continue
+ }
+ if got, want := dest.uint64s, tc.uint64s; !reflect.DeepEqual(got, want) {
+ t.Errorf("failed decode uint64s %d, got %v, want %v", i, got, want)
+ }
+ if got, want := dest.int64s, tc.int64s; !reflect.DeepEqual(got, want) {
+ t.Errorf("failed decode int64s %d, got %v, want %v", i, got, want)
+ }
+ }
+}
+
+type packedInts struct {
+ uint64s []uint64
+ int64s []int64
+}
+
+func (u *packedInts) decoder() []decoder {
+ return []decoder{
+ nil,
+ func(b *buffer, m message) error { return decodeUint64s(b, &m.(*packedInts).uint64s) },
+ func(b *buffer, m message) error { return decodeInt64s(b, &m.(*packedInts).int64s) },
+ }
+}
+
+func (u *packedInts) encode(b *buffer) {
+ encodeUint64s(b, 1, u.uint64s)
+ encodeInt64s(b, 2, u.int64s)
+}
diff --git a/framework/rest/profile/prune.go b/framework/rest/profile/prune.go
new file mode 100644
index 00000000..1924fada
--- /dev/null
+++ b/framework/rest/profile/prune.go
@@ -0,0 +1,97 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Implements methods to remove frames from profiles.
+
+package profile
+
+import (
+ "fmt"
+ "regexp"
+)
+
+// Prune removes all nodes beneath a node matching dropRx, and not
+// matching keepRx. If the root node of a Sample matches, the sample
+// will have an empty stack.
+func (p *Profile) Prune(dropRx, keepRx *regexp.Regexp) {
+ prune := make(map[uint64]bool)
+ pruneBeneath := make(map[uint64]bool)
+
+ for _, loc := range p.Location {
+ var i int
+ for i = len(loc.Line) - 1; i >= 0; i-- {
+ if fn := loc.Line[i].Function; fn != nil && fn.Name != "" {
+ funcName := fn.Name
+ // Account for leading '.' on the PPC ELF v1 ABI.
+ if funcName[0] == '.' {
+ funcName = funcName[1:]
+ }
+ if dropRx.MatchString(funcName) {
+ if keepRx == nil || !keepRx.MatchString(funcName) {
+ break
+ }
+ }
+ }
+ }
+
+ if i >= 0 {
+ // Found matching entry to prune.
+ pruneBeneath[loc.ID] = true
+
+ // Remove the matching location.
+ if i == len(loc.Line)-1 {
+ // Matched the top entry: prune the whole location.
+ prune[loc.ID] = true
+ } else {
+ loc.Line = loc.Line[i+1:]
+ }
+ }
+ }
+
+ // Prune locs from each Sample
+ for _, sample := range p.Sample {
+ // Scan from the root to the leaves to find the prune location.
+ // Do not prune frames before the first user frame, to avoid
+ // pruning everything.
+ foundUser := false
+ for i := len(sample.Location) - 1; i >= 0; i-- {
+ id := sample.Location[i].ID
+ if !prune[id] && !pruneBeneath[id] {
+ foundUser = true
+ continue
+ }
+ if !foundUser {
+ continue
+ }
+ if prune[id] {
+ sample.Location = sample.Location[i+1:]
+ break
+ }
+ if pruneBeneath[id] {
+ sample.Location = sample.Location[i:]
+ break
+ }
+ }
+ }
+}
+
+// RemoveUninteresting prunes and elides profiles using built-in
+// tables of uninteresting function names.
+func (p *Profile) RemoveUninteresting() error {
+ var keep, drop *regexp.Regexp
+ var err error
+
+ if p.DropFrames != "" {
+ if drop, err = regexp.Compile("^(" + p.DropFrames + ")$"); err != nil {
+ return fmt.Errorf("failed to compile regexp %s: %v", p.DropFrames, err)
+ }
+ if p.KeepFrames != "" {
+ if keep, err = regexp.Compile("^(" + p.KeepFrames + ")$"); err != nil {
+ return fmt.Errorf("failed to compile regexp %s: %v", p.KeepFrames, err)
+ }
+ }
+ p.Prune(drop, keep)
+ }
+ return nil
+}
diff --git a/framework/rest/server.go b/framework/rest/server.go
index 3fbca718..eb37a3a7 100644
--- a/framework/rest/server.go
+++ b/framework/rest/server.go
@@ -13,10 +13,10 @@ import (
register "github.com/unionj-cloud/go-doudou/v2/framework/registry"
"github.com/unionj-cloud/go-doudou/v2/framework/registry/constants"
"github.com/unionj-cloud/go-doudou/v2/framework/rest/httprouter"
+ "github.com/unionj-cloud/go-doudou/v2/toolkit/stringutils"
logger "github.com/unionj-cloud/go-doudou/v2/toolkit/zlogger"
"net"
"net/http"
- "net/http/pprof"
"os"
"os/signal"
"path"
@@ -117,11 +117,7 @@ func (srv *RestServer) printRoutes() {
_, ok := routes[key]
if !ok {
routes[key] = struct{}{}
- if strings.HasPrefix(r.Pattern, gddPathPrefix) || strings.HasPrefix(r.Pattern, debugPathPrefix) {
- data = append(data, []string{r.Name, r.Method, r.Pattern})
- } else {
- data = append(data, []string{r.Name, r.Method, path.Clean(config.GddConfig.RouteRootPath + r.Pattern)})
- }
+ data = append(data, []string{r.Name, r.Method, path.Clean(r.Pattern)})
}
}
tableString := &strings.Builder{}
@@ -202,7 +198,6 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
)
if config.GddConfig.ManageEnable {
srv.middlewares = append([]MiddlewareFunc{PrometheusMiddleware}, srv.middlewares...)
- gddRouter := srv.rootRouter.NewGroup(gddPathPrefix)
basicAuthMiddle := MiddlewareFunc(basicAuth())
gddmiddlewares := []MiddlewareFunc{metrics, basicAuthMiddle}
srv.gddRoutes = append(srv.gddRoutes, promRoutes()...)
@@ -216,7 +211,9 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
config.GddStatsFreq.Load(), err.Error(), config.DefaultGddStatsFreq)
freq, _ = time.ParseDuration(config.DefaultGddStatsFreq)
}
- _ = freq
+ statsvizServer, _ := statsviz.NewServer(statsviz.Root(srv.bizRouter.SubPath(gddPathPrefix+"statsviz/")), statsviz.SendFrequency(freq))
+ ws := statsvizServer.Ws()
+ index := statsvizServer.Index()
srv.gddRoutes = append(srv.gddRoutes, []Route{
{
Name: "GetStatsvizWs",
@@ -229,22 +226,23 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
Pattern: gddPathPrefix + "statsviz/*",
HandlerFunc: func(writer http.ResponseWriter, request *http.Request) {
if strings.HasSuffix(request.URL.Path, "/ws") {
- statsviz.Ws(writer, request)
+ ws(writer, request)
return
}
- statsviz.IndexAtRoot(gddPathPrefix+"statsviz/").ServeHTTP(writer, request)
+ index(writer, request)
},
},
}...)
- for _, item := range srv.gddRoutes {
- if item.HandlerFunc == nil {
- continue
- }
- h := http.Handler(item.HandlerFunc)
- for i := len(gddmiddlewares) - 1; i >= 0; i-- {
- h = gddmiddlewares[i].Middleware(h)
+ for k, item := range srv.gddRoutes {
+ if item.HandlerFunc != nil {
+ h := http.Handler(item.HandlerFunc)
+ for i := len(gddmiddlewares) - 1; i >= 0; i-- {
+ h = gddmiddlewares[i].Middleware(h)
+ }
+ srv.bizRouter.Handler(item.Method, item.Pattern, h, item.Name)
}
- gddRouter.Handler(item.Method, "/"+strings.TrimPrefix(item.Pattern, gddPathPrefix), h, item.Name)
+ item.Pattern = srv.bizRouter.SubPath(item.Pattern)
+ srv.gddRoutes[k] = item
}
srv.debugRoutes = append(srv.debugRoutes, []Route{
{
@@ -275,32 +273,32 @@ func NewRestServerWithOptions(options ...ServerOption) *RestServer {
lastSegment := request.URL.Path[strings.LastIndex(request.URL.Path, "/"):]
switch lastSegment {
case "/cmdline":
- pprof.Cmdline(writer, request)
+ Cmdline(writer, request)
return
case "/profile":
- pprof.Profile(writer, request)
+ Profile(writer, request)
return
case "/symbol":
- pprof.Symbol(writer, request)
+ Symbol(writer, request)
return
case "/trace":
- pprof.Trace(writer, request)
+ Trace(writer, request)
return
}
- pprof.Index(writer, request)
+ Index(writer, request)
},
},
}...)
- debugRouter := srv.rootRouter.NewGroup(debugPathPrefix)
- for _, item := range srv.debugRoutes {
- if item.HandlerFunc == nil {
- continue
- }
- h := http.Handler(item.HandlerFunc)
- for i := len(gddmiddlewares) - 1; i >= 0; i-- {
- h = gddmiddlewares[i].Middleware(h)
+ for k, item := range srv.debugRoutes {
+ if item.HandlerFunc != nil {
+ h := http.Handler(item.HandlerFunc)
+ for i := len(gddmiddlewares) - 1; i >= 0; i-- {
+ h = gddmiddlewares[i].Middleware(h)
+ }
+ srv.bizRouter.Handler(item.Method, item.Pattern, h, item.Name)
}
- debugRouter.Handler(item.Method, "/"+strings.TrimPrefix(item.Pattern, debugPathPrefix), h, item.Name)
+ item.Pattern = srv.bizRouter.SubPath(item.Pattern)
+ srv.debugRoutes[k] = item
}
}
srv.rootRouter.NotFound = http.HandlerFunc(http.NotFound)
@@ -322,6 +320,7 @@ func (srv *RestServer) groupRoutes(routeGroup *httprouter.RouteGroup, routes []R
h = mwf[i].Middleware(h)
}
routeGroup.Handler(item.Method, item.Pattern, h, item.Name)
+ item.Pattern = routeGroup.SubPath(item.Pattern)
srv.bizRoutes = append(srv.bizRoutes, item)
}
}
@@ -396,6 +395,24 @@ func (srv *RestServer) Run() {
}
func (srv *RestServer) Serve(ln net.Listener) {
+ var all []Route
+ all = append(all, srv.bizRoutes...)
+ all = append(all, srv.gddRoutes...)
+ all = append(all, srv.debugRoutes...)
+ for _, r := range all {
+ if strings.Contains(r.Pattern, gddPathPrefix+"doc") {
+ resources := strings.Split(strings.TrimPrefix(strings.TrimSuffix(r.Pattern, gddPathPrefix+"doc"), "/"), "/")
+ if len(resources) > 0 {
+ module := resources[len(resources)-1]
+ if stringutils.IsNotEmpty(module) {
+ Docs = append(Docs, DocItem{
+ Label: module,
+ Value: r.Pattern,
+ })
+ }
+ }
+ }
+ }
framework.PrintBanner()
framework.PrintLock.Lock()
register.NewRest(srv.data)
diff --git a/go.mod b/go.mod
index f188a880..efc17bf6 100644
--- a/go.mod
+++ b/go.mod
@@ -80,7 +80,7 @@ require (
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/nxadm/tail v1.4.8 // indirect
- github.com/rogpeppe/go-internal v1.10.0 // indirect
+ github.com/rogpeppe/go-internal v1.11.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
)
@@ -126,7 +126,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
- github.com/gorilla/websocket v1.4.2 // indirect
+ github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -197,7 +197,7 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
- golang.org/x/mod v0.8.0 // indirect
+ golang.org/x/mod v0.9.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sync v0.3.0
golang.org/x/sys v0.11.0 // indirect
@@ -214,7 +214,7 @@ require (
require (
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/apolloconfig/agollo/v4 v4.1.1-0.20220323095621-60ed86180f24
- github.com/arl/statsviz v0.5.1
+ github.com/arl/statsviz v0.6.0
github.com/armon/go-metrics v0.4.1
github.com/containerd/containerd v1.5.18 // indirect
github.com/docker/docker v20.10.7+incompatible
diff --git a/go.sum b/go.sum
index bf8d4936..4e79134d 100644
--- a/go.sum
+++ b/go.sum
@@ -129,8 +129,8 @@ github.com/antlr/antlr4 v0.0.0-20200124162019-2d7f727a00b7 h1:4IkFZAFQ87SeXXF6n+
github.com/antlr/antlr4 v0.0.0-20200124162019-2d7f727a00b7/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y=
github.com/apolloconfig/agollo/v4 v4.1.1-0.20220323095621-60ed86180f24 h1:ZMK8YXD82znWnBrn9XF7wlPFDXPdBrPphL6ZS2pIKqQ=
github.com/apolloconfig/agollo/v4 v4.1.1-0.20220323095621-60ed86180f24/go.mod h1:SuvTjtg0p4UlSzSbik+ibLRr6oR1xRsfy65QzP3GEAs=
-github.com/arl/statsviz v0.5.1 h1:3HY0ZEB738JtguWsD1Tf1pFJZiCcWUmYRq/3OTYKaSI=
-github.com/arl/statsviz v0.5.1/go.mod h1:zDnjgRblGm1Dyd7J5YlbH7gM1/+HRC+SfkhZhQb5AnM=
+github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE=
+github.com/arl/statsviz v0.6.0/go.mod h1:0toboo+YGSUXDaS4g1D5TVS4dXs7S7YYT5J/qnW2h8s=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -641,8 +641,9 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -1115,8 +1116,9 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
@@ -1417,8 +1419,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
diff --git a/version/version.go b/version/version.go
index 3dbde87d..2a3212b8 100644
--- a/version/version.go
+++ b/version/version.go
@@ -1,3 +1,3 @@
package version
-const Release = "v2.3.3"
+const Release = "v2.3.4"