From 288c439171972069858accb6e87f4009d1a9b452 Mon Sep 17 00:00:00 2001 From: Andrei Fedotov Date: Wed, 13 Mar 2024 17:20:15 +0300 Subject: [PATCH] Added stable test. Renamed options. Fix docs. Signed-off-by: Andrey Fedotov --- contrib/tester-progs/Makefile | 6 +++- contrib/tester-progs/go/getcpu/getcpu.go | 2 -- .../go/user-stacktrace/user-stacktrace.go | 29 +++++++++++++++++ .../docs/concepts/tracing-policy/selectors.md | 12 +++++-- docs/data/tetragon_flags.yaml | 2 +- pkg/encoder/encoder.go | 8 +++-- pkg/option/flags.go | 6 ++-- pkg/sensors/tracing/kprobe_test.go | 31 +++++++++++-------- 8 files changed, 71 insertions(+), 25 deletions(-) create mode 100644 contrib/tester-progs/go/user-stacktrace/user-stacktrace.go diff --git a/contrib/tester-progs/Makefile b/contrib/tester-progs/Makefile index 77ff318a6eb..1e3f43c968e 100644 --- a/contrib/tester-progs/Makefile +++ b/contrib/tester-progs/Makefile @@ -22,7 +22,8 @@ PROGS = sigkill-tester \ drop-privileges \ getcpu \ direct-write-tester \ - change-capabilities + change-capabilities \ + user-stacktrace # For now enforcer-tester is compiled to 32-bit only on x86_64 as we want # to test 32-bit binaries and system calls compatibility layer. @@ -88,6 +89,9 @@ lseek-pipe: FORCE getcpu: FORCE go build -o getcpu ./go/getcpu +user-stacktrace: FORCE + go build -o user-stacktrace ./go/user-stacktrace + .PHONY: clean clean: rm -f $(PROGS) diff --git a/contrib/tester-progs/go/getcpu/getcpu.go b/contrib/tester-progs/go/getcpu/getcpu.go index c9fdf47f575..40176cfdba1 100644 --- a/contrib/tester-progs/go/getcpu/getcpu.go +++ b/contrib/tester-progs/go/getcpu/getcpu.go @@ -5,7 +5,6 @@ package main import ( "os" - "time" "unsafe" "golang.org/x/sys/unix" @@ -19,6 +18,5 @@ func main() { uintptr(unsafe.Pointer(&node)), 0, ) - time.Sleep(1 * time.Second) os.Exit(int(err)) } diff --git a/contrib/tester-progs/go/user-stacktrace/user-stacktrace.go b/contrib/tester-progs/go/user-stacktrace/user-stacktrace.go new file mode 100644 index 00000000000..aba61ccdf54 --- /dev/null +++ b/contrib/tester-progs/go/user-stacktrace/user-stacktrace.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Tetragon + +package main + +import ( + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +//nolint:all +func main() { + // This test has endless loop for stable stack trace collection. + // It must be terminated from tests. + var cpu, node int + _, _, err := unix.Syscall( + unix.SYS_GETCPU, + uintptr(unsafe.Pointer(&cpu)), + uintptr(unsafe.Pointer(&node)), + 0, + ) + if err != 0 { + os.Exit(int(err)) + } + for { + } +} diff --git a/docs/content/en/docs/concepts/tracing-policy/selectors.md b/docs/content/en/docs/concepts/tracing-policy/selectors.md index f33c3bdee1a..d2adfcf8c2e 100644 --- a/docs/content/en/docs/concepts/tracing-policy/selectors.md +++ b/docs/content/en/docs/concepts/tracing-policy/selectors.md @@ -851,7 +851,7 @@ kprobes: {{< caution >}} By default Tetragon does not expose the linear addresses from kernel space or -user space, you need to enable the flag `--expose-addresses` to get the addresses +user space, you need to enable the flag `--expose-stack-addresses` to get the addresses along the rest. Note that the Tetragon agent is using its privilege to read the kernel symbols @@ -914,6 +914,14 @@ beginning of the binary module. "module" is the absolute path of the binary file to which address belongs. "symbol" is the function symbol name. "symbol" may be missing if the binary file is stripped. +{{< note >}} +Information from `procfs (/proc//maps)` is used to symbolize user +stack trace addresses. Stack trace addresses extraction and symbolizing are async. +It might happen that process is terminated and the `/proc//maps` file will be +not existed at user stack trace symbolization step. In such case user stack traces +for very short living process might be not collected. +{{< /note >}} + This output can be enhanced in a more human friendly using the `tetra getevents -o compact` command. Indeed, by default, it will print the stack trace along the compact output of the event similarly to this: @@ -938,7 +946,7 @@ The printing format for user stack trace is `"0x%x: %s (%s+0x%x)", address, symb {{< note >}} Compact output will display missing addresses as `0x0`, see the above note on -`--expose-addresses` for more info. +`--expose-stack-addresses` for more info. {{< /note >}} ### NoPost action diff --git a/docs/data/tetragon_flags.yaml b/docs/data/tetragon_flags.yaml index 6d85c8cd10d..a70f6808e14 100644 --- a/docs/data/tetragon_flags.yaml +++ b/docs/data/tetragon_flags.yaml @@ -94,7 +94,7 @@ options: default_value: "-1" usage: | Rate limit (per minute) for event export. Set to -1 to disable - - name: expose-addresses + - name: expose-stack-addresses default_value: "false" usage: Expose real linear addresses in events stack traces - name: field-filters diff --git a/pkg/encoder/encoder.go b/pkg/encoder/encoder.go index baf8172a9d5..a2663d69646 100644 --- a/pkg/encoder/encoder.go +++ b/pkg/encoder/encoder.go @@ -211,11 +211,13 @@ func HumanStackTrace(response *tetragon.GetEventsResponse, colorer *Colorer) str } } if ev.ProcessKprobe.UserStackTrace != nil { - fmt.Fprintf(out, "User space:\n") + fmt.Fprintf(out, "User:\n") for _, st := range ev.ProcessKprobe.UserStackTrace { colorer.Green.Fprintf(out, " 0x%x:", st.Address) - colorer.Blue.Fprintf(out, " %s ", st.Symbol) - colorer.Yellow.Fprintf(out, "(%s+0x%x)\n", st.Module, st.Offset) + if st.Symbol != "" { + colorer.Blue.Fprintf(out, " %s", st.Symbol) + } + colorer.Yellow.Fprintf(out, " (%s+0x%x)\n", st.Module, st.Offset) } } } diff --git a/pkg/option/flags.go b/pkg/option/flags.go index 60093181528..0d9b8170b84 100644 --- a/pkg/option/flags.go +++ b/pkg/option/flags.go @@ -89,7 +89,7 @@ const ( KeyEnablePodInfo = "enable-pod-info" KeyEnableTracingPolicyCRD = "enable-tracing-policy-crd" - KeyExposeAddresses = "expose-addresses" + KeyExposeStackAddresses = "expose-stack-addresses" KeyGenerateDocs = "generate-docs" ) @@ -170,7 +170,7 @@ func ReadAndSetFlags() error { Config.TracingPolicy = viper.GetString(KeyTracingPolicy) - Config.ExposeAddresses = viper.GetBool(KeyExposeAddresses) + Config.ExposeAddresses = viper.GetBool(KeyExposeStackAddresses) return nil } @@ -275,7 +275,7 @@ func AddFlags(flags *pflag.FlagSet) { flags.Bool(KeyEnablePodInfo, false, "Enable PodInfo custom resource") flags.Bool(KeyEnableTracingPolicyCRD, true, "Enable TracingPolicy and TracingPolicyNamespaced custom resources") - flags.Bool(KeyExposeAddresses, false, "Expose real linear addresses in events stack traces") + flags.Bool(KeyExposeStackAddresses, false, "Expose real linear addresses in events stack traces") flags.Bool(KeyGenerateDocs, false, "Generate documentation in YAML format to stdout") } diff --git a/pkg/sensors/tracing/kprobe_test.go b/pkg/sensors/tracing/kprobe_test.go index 8174e893565..9d972aa67e2 100644 --- a/pkg/sensors/tracing/kprobe_test.go +++ b/pkg/sensors/tracing/kprobe_test.go @@ -6148,7 +6148,7 @@ func TestKprobeUserStackTrace(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), tus.Conf().CmdWaitTime) defer cancel() - testGetCpu := testutils.RepoRootPath("contrib/tester-progs/getcpu") + testUserStacktrace := testutils.RepoRootPath("contrib/tester-progs/user-stacktrace") tracingPolicy := `apiVersion: cilium.io/v1alpha1 kind: TracingPolicy metadata: @@ -6160,7 +6160,7 @@ spec: - matchBinaries: - operator: "In" values: - - "` + testGetCpu + `" + - "` + testUserStacktrace + `" matchActions: - action: Post userStackTrace: true` @@ -6173,27 +6173,32 @@ spec: } observertesthelper.LoopEvents(ctx, t, &doneWG, &readyWG, obs) readyWG.Wait() + test_cmd := exec.Command(testUserStacktrace) - if err := exec.Command(testGetCpu).Run(); err != nil { - t.Fatalf("failed to run %s: %s", testGetCpu, err) + if err := test_cmd.Start(); err != nil { + t.Fatalf("failed to run %s: %s", testUserStacktrace, err) } stackTraceChecker := ec.NewProcessKprobeChecker("user-stack-trace"). - WithProcess(ec.NewProcessChecker().WithBinary(sm.Full(testGetCpu))). + WithProcess(ec.NewProcessChecker().WithBinary(sm.Full(testUserStacktrace))). WithUserStackTrace(ec.NewStackTraceEntryListMatcher().WithValues( ec.NewStackTraceEntryChecker().WithSymbol(sm.Suffix(("main.main"))), - // syscall user-nix /home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu __x64_sys_getcpu - // User space: - // 0x0: runtime/internal/syscall.Syscall6 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x2aee) - // 0x0: syscall.Syscall (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x635e6) - // 0x0: syscall.Syscall.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x6374e) - // 0x0: main.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x652e5) - // 0x0: runtime.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x3313d) - // 0x0: runtime.goexit.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x5e901) + // syscall user-nix /home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace __x64_sys_getcpu + // User: + // 0x0: runtime/internal/syscall.Syscall6 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x2aee) + // 0x0: syscall.Syscall (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x63346) + // 0x0: syscall.Syscall.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x634ae) + // 0x0: main.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x6503e) + // 0x0: runtime.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x3313d) + // 0x0: runtime.goexit.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x5e661) )) checker := ec.NewUnorderedEventChecker(stackTraceChecker) err = jsonchecker.JsonTestCheck(t, checker) + + // Kill test because of endless loop in the test for stable stack trace extraction + test_cmd.Process.Kill() + assert.NoError(t, err) }