diff --git a/orbit/changes/8485-dump-pprof b/orbit/changes/8485-dump-pprof new file mode 100644 index 0000000000..8f6a2a5165 --- /dev/null +++ b/orbit/changes/8485-dump-pprof @@ -0,0 +1,2 @@ +- On Unix systems, dump pprof data into a `profiles` directory in the orbit root dir + when receiving a SIGUSR1. This is to assist debugging for memory leaks diff --git a/orbit/cmd/orbit/orbit.go b/orbit/cmd/orbit/orbit.go index 429d98e32a..d79d98bec5 100644 --- a/orbit/cmd/orbit/orbit.go +++ b/orbit/cmd/orbit/orbit.go @@ -666,6 +666,8 @@ func main() { defer cancel() g.Add(signalHandler(ctx)) + go sigusrListener(c.String("root-dir")) + if err := g.Run(); err != nil { log.Error().Err(err).Msg("unexpected exit") } diff --git a/orbit/cmd/orbit/signal_unix.go b/orbit/cmd/orbit/signal_unix.go index f8e7c0f2b4..2417a0c9bd 100644 --- a/orbit/cmd/orbit/signal_unix.go +++ b/orbit/cmd/orbit/signal_unix.go @@ -3,13 +3,86 @@ package main import ( + "archive/tar" + "bytes" + "compress/gzip" "context" + "fmt" "os" + "os/signal" + "path" + "runtime/pprof" "syscall" + "time" + "github.com/fleetdm/fleet/v4/orbit/pkg/constant" + "github.com/fleetdm/fleet/v4/pkg/secure" "github.com/oklog/run" + "github.com/rs/zerolog/log" ) func signalHandler(ctx context.Context) (execute func() error, interrupt func(error)) { return run.SignalHandler(ctx, os.Interrupt, os.Kill, syscall.SIGTERM) } + +func sigusrListener(rootDir string) { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGUSR1) + for { + <-c + + if err := dumpProf(rootDir); err != nil { + log.Warn().Err(err).Msg("unable to create pprof") + } + } +} + +func dumpProf(rootDir string) error { + if err := secure.MkdirAll(path.Join(rootDir, "profiles"), constant.DefaultDirMode); err != nil { + return err + } + now := time.Now() + + // We can't use ISO 8601/RFC 3339 because NTFS and FAT do not allow colons in filenames + timestamp := now.UTC().Format("2006-01-02T15-04-05") + + out, err := os.Create(path.Join(rootDir, "profiles", fmt.Sprintf("profiles-%s.tar.gz", timestamp))) + if err != nil { + return err + } + defer out.Close() + + gw := gzip.NewWriter(out) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + buf := new(bytes.Buffer) + + for _, profile := range pprof.Profiles() { + err = profile.WriteTo(buf, 0) + if err != nil { + return err + } + + header := tar.Header{ + Typeflag: tar.TypeReg, + Name: fmt.Sprintf("%s.pprof", profile.Name()), + Size: int64(buf.Len()), + Mode: 0o664, + ModTime: now, + AccessTime: now, + ChangeTime: now, + } + err = tw.WriteHeader(&header) + if err != nil { + return err + } + _, err = buf.WriteTo(tw) + if err != nil { + return err + } + buf.Reset() + } + return nil +} diff --git a/orbit/cmd/orbit/signal_windows.go b/orbit/cmd/orbit/signal_windows.go index 4e9e2a08a1..dacd405e11 100644 --- a/orbit/cmd/orbit/signal_windows.go +++ b/orbit/cmd/orbit/signal_windows.go @@ -10,3 +10,7 @@ import ( func signalHandler(ctx context.Context) (execute func() error, interrupt func(error)) { return run.SignalHandler(ctx, os.Interrupt, os.Kill) } + +func sigusrListener(rootDir string) error { + return nil +}