From 2071a1ed8d39e59d94e37dcac5c4ff36ece206ae Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Mon, 26 Apr 2021 18:17:24 -0700 Subject: [PATCH] Truncate database during open if necessary (#20) In certain non-clean shutdown scenarios, Badger will refuse to start up without calling truncate. If this is necessary, do so and log about possible data loss. Actual data loss is unlikely at this point due to limited use of the Badger database. --- cmd/orbit/orbit.go | 14 ++++++++++++-- go.mod | 1 + go.sum | 6 ++++++ pkg/database/database.go | 22 +++++++++++++++++++++- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/cmd/orbit/orbit.go b/cmd/orbit/orbit.go index 8ff79f8aa0..9cfdd2224d 100644 --- a/cmd/orbit/orbit.go +++ b/cmd/orbit/orbit.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/dgraph-io/badger" "github.com/fleetdm/orbit/pkg/certificate" "github.com/fleetdm/orbit/pkg/constant" "github.com/fleetdm/orbit/pkg/database" @@ -151,9 +152,18 @@ func main() { return errors.Wrap(err, "initialize root dir") } - db, err := database.Open(filepath.Join(c.String("root-dir"), "orbit.db")) + dbPath := filepath.Join(c.String("root-dir"), "orbit.db") + db, err := database.Open(dbPath) if err != nil { - return err + if errors.Is(err, badger.ErrTruncateNeeded) { + db, err = database.OpenTruncate(dbPath) + if err != nil { + return err + } + log.Warn().Msg("Open badger required truncate. Data loss is possible.") + } else { + return err + } } defer func() { if err := db.Close(); err != nil { diff --git a/go.mod b/go.mod index df417fe636..c5bf234e87 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/fleetdm/orbit go 1.15 require ( + github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/badger/v2 v2.2007.2 github.com/fatih/color v1.10.0 github.com/goreleaser/nfpm/v2 v2.2.2 diff --git a/go.sum b/go.sum index 270fb6fe89..bb19dbf2fa 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= @@ -127,8 +129,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218= github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -414,6 +419,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= diff --git a/pkg/database/database.go b/pkg/database/database.go index 100377348d..ac3481b54b 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -22,7 +22,7 @@ type BadgerDB struct { closeChan chan struct{} } -// Open opens (initializing if necessary) a new Badger database at the specified +// Open opens (initializing if necessary) a Badger database at the specified // path. Users must close the DB with Close(). func Open(path string) (*BadgerDB, error) { // DefaultOptions sets synchronous writes to true (maximum data integrity). @@ -38,6 +38,26 @@ func Open(path string) (*BadgerDB, error) { return b, nil } +// OpenTruncate opens (initializing and/or truncating if necessary) a Badger +// database at the specified path. Users must close the DB with Close(). +// +// Prefer Open in the general case, but after a bad shutdown it may be necessary +// to call OpenTruncate. This may cause data loss. Detect this situation by +// looking for badger.ErrTruncateNeeded. +func OpenTruncate(path string) (*BadgerDB, error) { + // DefaultOptions sets synchronous writes to true (maximum data integrity). + // TODO implement logging? + db, err := badger.Open(badger.DefaultOptions(path).WithLogger(nil).WithTruncate(true)) + if err != nil { + return nil, errors.Wrapf(err, "open badger with truncate %s", path) + } + + b := &BadgerDB{DB: db} + b.startBackgroundCompaction() + + return b, nil +} + // startBackgroundCompaction starts a background loop that will call the // compaction method on the database. Badger does not do this automatically, so // we need to be sure to do so here (or elsewhere).