// Package ctxerr provides functions to wrap errors with annotations and // stack traces, and to handle those errors such that unique instances of // those errors will be stored for an amount of time so that it can be // used to troubleshoot issues. // // Typical uses of this package should be to call New or Wrap[f] as close as // possible from where the error is encountered (or where it needs to be // created for New), and then to call Handle with the error only once, after it // bubbled back to the top of the call stack (e.g. in the HTTP handler, or in // the CLI command, etc.). It is fine to wrap the error with more annotations // along the way, by calling Wrap[f]. package ctxerr import ( "context" "encoding/json" "errors" "fmt" "runtime" "strings" "time" "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/getsentry/sentry-go" "go.elastic.co/apm/v2" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" ) type key int const errHandlerKey key = 0 // Defining here for testing purposes var nowFn = time.Now // FleetError is the error implementation used by this package. type FleetError struct { msg string // error message to be prepended to cause stack stackTracer // stack trace where this error was created cause error // original error that caused this error if non-nil data json.RawMessage // additional metadata about the error (timestamps, etc) } type fleetErrorJSON struct { Message string `json:"message,omitempty"` Data json.RawMessage `json:"data,omitempty"` Stack []string `json:"stack,omitempty"` } // Error implements the error interface. func (e FleetError) Error() string { if e.cause == nil { return e.msg } return fmt.Sprintf("%s: %s", e.msg, e.cause.Error()) } // Unwrap implements the error Unwrap interface introduced in go1.13. func (e *FleetError) Unwrap() error { return e.cause } // Stack returns a call stack for the error func (e *FleetError) Stack() []string { return e.stack.List() } // StackTrace implements the runtimeStackTracer interface understood by the // elastic APM package to reuse already-captured stack traces. // https://github.com/elastic/apm-agent-go/blob/main/stacktrace/errors.go#L45-L47 func (e *FleetError) StackTrace() *runtime.Frames { st := e.stack.(stack) // outside of tests, e.stack is always a stack type return runtime.CallersFrames(st) } // StackFrames implements the reflection-based method that Sentry's Go SDK // uses to look for a stack trace. It abuses the internals a bit, as it uses // the name that sentry looks for, but returns the []uintptr slice (which works // because of how they handle the returned value via reflection). A cleaner // approach would be if they used an interface detection like APM does. // https://github.com/getsentry/sentry-go/blob/master/stacktrace.go#L44-L49 func (e *FleetError) StackFrames() []uintptr { return e.stack.(stack) // outside of tests, e.stack is always a stack type } // LogFields implements fleet.ErrWithLogFields, so attached error data can be // logged along with the error func (e *FleetError) LogFields() []any { var fields []any var data map[string]any if len(e.data) == 0 { return fields } // if we fail to unmarshal the data, return it as a raw string. It // won't be as easy to read but it will be there. if err := json.Unmarshal(e.data, &data); err != nil { return []any{ "data", string(e.data), } } for k, v := range data { fields = append(fields, k, v) } return fields } // setMetadata adds common metadata attributes to the `data` map provided. // NOTE: this will mutate the data provided and override other values with the same keys. func setMetadata(ctx context.Context, data map[string]interface{}) map[string]interface{} { if data == nil { data = map[string]interface{}{} } data["timestamp"] = nowFn().Format(time.RFC3339) if h, ok := host.FromContext(ctx); ok { data["host"] = map[string]interface{}{ "platform": h.Platform, "osquery_version": h.OsqueryVersion, } } if v, ok := viewer.FromContext(ctx); ok { vdata := map[string]interface{}{} data["viewer"] = vdata vdata["is_logged_in"] = v.IsLoggedIn() if v.User != nil { vdata["sso_enabled"] = v.User.SSOEnabled } } return data } func encodeData(ctx context.Context, data map[string]interface{}, augment bool) json.RawMessage { if augment { data = setMetadata(ctx, data) } encoded, err := json.Marshal(data) if err != nil { msg := fmt.Sprintf(`{"error": "there was an error encoding additional data: %s"}`, err.Error()) encoded = json.RawMessage(msg) } return encoded } func newError(ctx context.Context, msg string, cause error, data map[string]interface{}) error { stack := newStack(2) edata := encodeData(ctx, data, true) return &FleetError{msg, stack, cause, edata} } func wrapError(ctx context.Context, msg string, cause error, data map[string]interface{}) error { if cause == nil { return nil } stack := newStack(2) var ferr *FleetError isFleetError := errors.As(cause, &ferr) // If the error is a FleetError, don't add the full stack trace as it should // already be present. if isFleetError { stack = stack[:1] } edata := encodeData(ctx, data, !isFleetError) return &FleetError{msg, stack, cause, edata} } // New creates a new error with the given message. func New(ctx context.Context, msg string) error { return newError(ctx, msg, nil, nil) } // NewWithData creates a new error and attaches additional metadata to it func NewWithData(ctx context.Context, msg string, data map[string]interface{}) error { return newError(ctx, msg, nil, data) } // Errorf creates a new error with the given message. func Errorf(ctx context.Context, format string, args ...interface{}) error { msg := fmt.Sprintf(format, args...) return newError(ctx, msg, nil, nil) } // Wrap creates a new error with the given message, wrapping another error. func Wrap(ctx context.Context, cause error, msgs ...string) error { msg := strings.Join(msgs, " ") return wrapError(ctx, msg, cause, nil) } // WrapWithData creates a new error with the given message, wrapping another // error and attaching the data provided to it. func WrapWithData(ctx context.Context, cause error, msg string, data map[string]interface{}) error { return wrapError(ctx, msg, cause, data) } // Wrapf creates a new error with the given message, wrapping another error. func Wrapf(ctx context.Context, cause error, format string, args ...interface{}) error { msg := fmt.Sprintf(format, args...) return wrapError(ctx, msg, cause, nil) } // Cause returns the root error in err's chain. func Cause(err error) error { return fleet.Cause(err) } // FleetCause is similar to Cause, but returns the root-most // FleetError in the chain func FleetCause(err error) *FleetError { var ferr, aux *FleetError var ok bool for err != nil { if aux, ok = err.(*FleetError); ok { ferr = aux } err = Unwrap(err) } return ferr } // Unwrap is a wrapper of built-in errors.Unwrap. It returns the result of // calling the Unwrap method on err, if err's type contains an Unwrap method // returning error. Otherwise, Unwrap returns nil. func Unwrap(err error) error { return errors.Unwrap(err) } // MarshalJSON provides a JSON representation of a whole error chain. func MarshalJSON(err error) ([]byte, error) { chain := make([]fleetErrorJSON, 0) for err != nil { switch v := err.(type) { case *FleetError: chain = append(chain, fleetErrorJSON{ Message: v.msg, Data: v.data, Stack: v.stack.List(), }) default: chain = append(chain, fleetErrorJSON{Message: v.Error()}) } err = Unwrap(err) } // reverse the chain to present errors in chronological order. for i := len(chain)/2 - 1; i >= 0; i-- { opp := len(chain) - 1 - i chain[i], chain[opp] = chain[opp], chain[i] } return json.MarshalIndent(chain, "", " ") } // StoredError represents the structure we use to de-serialize errors and // counts stored in Redis type StoredError struct { Count int `json:"count"` Chain json.RawMessage `json:"chain"` } type Handler interface { Store(error) Retrieve(flush bool) ([]*StoredError, error) } // NewContext returns a context derived from ctx that contains the provided // error handler. func NewContext(ctx context.Context, eh Handler) context.Context { return context.WithValue(ctx, errHandlerKey, eh) } func FromContext(ctx context.Context) Handler { v, _ := ctx.Value(errHandlerKey).(Handler) return v } // Handle handles err by passing it to the registered error handler, // deduplicating it and storing it for a configured duration. It also takes // care of sending it to the configured OpenTelemetry/APM/Sentry, if any. func Handle(ctx context.Context, err error) { if err == nil { return } // as a last resource, wrap the error if there isn't // a FleetError in the chain var ferr *FleetError if !errors.As(err, &ferr) { wrapped := wrapError(ctx, "missing FleetError in chain", err, nil) if wrapped == nil { return // shouldn't happen since err is not nil, but be safe } // wrapError returns an error interface, but we know it's a *FleetError ferr = wrapped.(*FleetError) } cause := ferr if rootCause := FleetCause(ferr); rootCause != nil { // use the FleetCause error so we send the most relevant stacktrace to APM // (the one from the initial New/Wrap call). cause = rootCause } // send to OpenTelemetry if there's an active span if span := trace.SpanFromContext(ctx); span != nil && span.IsRecording() { // Mark the current span as failed by setting the error status. // This status can be overridden if we recovered from the error. span.SetStatus(codes.Error, cause.Error()) // Build attributes for the exception event attrs := []attribute.KeyValue{ attribute.String("exception.type", fmt.Sprintf("%T", cause)), attribute.String("exception.message", cause.Error()), attribute.String("exception.stacktrace", strings.Join(cause.Stack(), "\n")), } // Add contextual information if available (same as Sentry) v, _ := viewer.FromContext(ctx) h, _ := host.FromContext(ctx) if v.User != nil { attrs = append(attrs, // Not sending the email here as it may contain sensitive information (PII). attribute.Int64("user.id", int64(v.User.ID)), //nolint:gosec ) } else if h != nil { attrs = append(attrs, attribute.String("host.hostname", h.Hostname), attribute.Int64("host.id", int64(h.ID)), //nolint:gosec ) } span.AddEvent("exception", trace.WithAttributes(attrs...)) } // send to elastic APM apm.CaptureError(ctx, cause).Send() // if Sentry is configured, capture the error there if sentryClient := sentry.CurrentHub().Client(); sentryClient != nil { // sentry is configured, add contextual information if available v, _ := viewer.FromContext(ctx) h, _ := host.FromContext(ctx) if v.User != nil || h != nil { // we have a viewer (user) or a host in the context, use this to // enrich the error with more context ctxHub := sentry.CurrentHub().Clone() if v.User != nil { ctxHub.ConfigureScope(func(scope *sentry.Scope) { scope.SetTag("email", v.User.Email) scope.SetTag("user_id", fmt.Sprint(v.User.ID)) }) } else if h != nil { ctxHub.ConfigureScope(func(scope *sentry.Scope) { scope.SetTag("hostname", h.Hostname) scope.SetTag("host_id", fmt.Sprint(h.ID)) }) } ctxHub.CaptureException(cause) } else { sentry.CaptureException(cause) } } if eh := FromContext(ctx); eh != nil { eh.Store(ferr) } } // Retrieve retrieves an error from the registered error handler func Retrieve(ctx context.Context) ([]*StoredError, error) { eh := FromContext(ctx) if eh == nil { return nil, New(ctx, "missing handler in context") } return eh.Retrieve(false) } // MockHandler is a mock implementation of an error handler that allows to test // ctxerr features that retrieve and store information in Redis without a // server running. // Ideally this should live in errorstore/errors, but that creates a circular // dependency. type MockHandler struct { StoreImpl func(err error) RetrieveImpl func(flush bool) ([]*StoredError, error) } func (h MockHandler) Store(err error) { h.StoreImpl(err) } func (h MockHandler) Retrieve(flush bool) ([]*StoredError, error) { return h.RetrieveImpl(flush) }