mirror of
https://github.com/fleetdm/fleet
synced 2026-05-24 09:28:54 +00:00
506901443d
<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #37244 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. ## Testing - [x] QA'd all new/changed functionality manually <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Refactor** * Internal MySQL utility package reorganized and all internal imports updated to the new platform location; no changes to end-user functionality or behavior. * **Documentation** * Added platform package documentation describing infrastructure responsibilities and architectural boundaries to guide maintainers. <sub>✏️ Tip: You can customize this high-level summary in your review settings.</sub> <!-- end of auto-generated comment: release notes by coderabbit.ai -->
102 lines
2.9 KiB
Go
102 lines
2.9 KiB
Go
package mysql
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"time"
|
|
|
|
"github.com/VividCortex/mysqlerr"
|
|
"github.com/cenkalti/backoff/v4"
|
|
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
|
|
"github.com/go-kit/log"
|
|
"github.com/go-sql-driver/mysql"
|
|
"github.com/jmoiron/sqlx"
|
|
)
|
|
|
|
var DoRetryErr = errors.New("fleet datastore retry")
|
|
|
|
type TxFn func(tx sqlx.ExtContext) error
|
|
|
|
// ReadTxFn is the read-only variant of TxFn, with tx only exposing the read methods
|
|
type ReadTxFn func(tx DBReadTx) error
|
|
|
|
// WithRetryTxx provides a common way to commit/rollback a txFn wrapped in a retry with exponential backoff
|
|
func WithRetryTxx(ctx context.Context, db *sqlx.DB, fn TxFn, logger log.Logger) error {
|
|
operation := func() error {
|
|
tx, err := db.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return ctxerr.Wrap(ctx, err, "create transaction")
|
|
}
|
|
|
|
defer func() {
|
|
if p := recover(); p != nil {
|
|
if err := tx.Rollback(); err != nil {
|
|
logger.Log("err", err, "msg", "error encountered during transaction panic rollback")
|
|
}
|
|
panic(p)
|
|
}
|
|
}()
|
|
|
|
if err := fn(tx); err != nil {
|
|
rbErr := tx.Rollback()
|
|
if rbErr != nil && rbErr != sql.ErrTxDone {
|
|
// Consider rollback errors to be non-retryable
|
|
return backoff.Permanent(ctxerr.Wrapf(ctx, err, "got err '%s' rolling back after err", rbErr.Error()))
|
|
}
|
|
|
|
if retryableError(err) {
|
|
return err
|
|
}
|
|
|
|
// Consider any other errors to be non-retryable
|
|
return backoff.Permanent(err)
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
err = ctxerr.Wrap(ctx, err, "commit transaction")
|
|
|
|
if retryableError(err) {
|
|
return err
|
|
}
|
|
|
|
return backoff.Permanent(err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
expBo := backoff.NewExponentialBackOff()
|
|
// MySQL innodb_lock_wait_timeout default is 50 seconds, so transaction can be waiting for a lock for several seconds.
|
|
// Setting a higher MaxElapsedTime to increase probability that transaction will be retried.
|
|
// This will reduce the number of retryable 'Deadlock found' errors. However, with a loaded DB, we will still see
|
|
// 'Context cancelled' errors when the server drops long-lasting connections.
|
|
expBo.MaxElapsedTime = 1 * time.Minute
|
|
bo := backoff.WithMaxRetries(expBo, 5)
|
|
return backoff.Retry(operation, bo)
|
|
}
|
|
|
|
// RetryableError determines whether a MySQL error can be retried. By default
|
|
// errors are considered non-retryable. Only errors that we know have a
|
|
// possibility of succeeding on a retry should return true in this function.
|
|
func RetryableError(err error) bool {
|
|
base := ctxerr.Cause(err)
|
|
if b, ok := base.(*mysql.MySQLError); ok {
|
|
switch b.Number {
|
|
// Consider lock related errors to be retryable
|
|
case mysqlerr.ER_LOCK_DEADLOCK, mysqlerr.ER_LOCK_WAIT_TIMEOUT:
|
|
return true
|
|
}
|
|
}
|
|
if errors.Is(err, DoRetryErr) {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// retryableError is the internal (non-exported) version that calls RetryableError.
|
|
// Kept for backwards compatibility with existing callers in this package.
|
|
func retryableError(err error) bool {
|
|
return RetryableError(err)
|
|
}
|