diff --git a/cmd/orbit/orbit.go b/cmd/orbit/orbit.go index 8ff79f8aa0..9cfdd2224d 100644 --- a/cmd/orbit/orbit.go +++ b/cmd/orbit/orbit.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/dgraph-io/badger" "github.com/fleetdm/orbit/pkg/certificate" "github.com/fleetdm/orbit/pkg/constant" "github.com/fleetdm/orbit/pkg/database" @@ -151,9 +152,18 @@ func main() { return errors.Wrap(err, "initialize root dir") } - db, err := database.Open(filepath.Join(c.String("root-dir"), "orbit.db")) + dbPath := filepath.Join(c.String("root-dir"), "orbit.db") + db, err := database.Open(dbPath) if err != nil { - return err + if errors.Is(err, badger.ErrTruncateNeeded) { + db, err = database.OpenTruncate(dbPath) + if err != nil { + return err + } + log.Warn().Msg("Open badger required truncate. Data loss is possible.") + } else { + return err + } } defer func() { if err := db.Close(); err != nil { diff --git a/go.mod b/go.mod index df417fe636..c5bf234e87 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/fleetdm/orbit go 1.15 require ( + github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/badger/v2 v2.2007.2 github.com/fatih/color v1.10.0 github.com/goreleaser/nfpm/v2 v2.2.2 diff --git a/go.sum b/go.sum index 270fb6fe89..bb19dbf2fa 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= @@ -127,8 +129,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218= github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -414,6 +419,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= diff --git a/pkg/database/database.go b/pkg/database/database.go index 100377348d..ac3481b54b 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -22,7 +22,7 @@ type BadgerDB struct { closeChan chan struct{} } -// Open opens (initializing if necessary) a new Badger database at the specified +// Open opens (initializing if necessary) a Badger database at the specified // path. Users must close the DB with Close(). func Open(path string) (*BadgerDB, error) { // DefaultOptions sets synchronous writes to true (maximum data integrity). @@ -38,6 +38,26 @@ func Open(path string) (*BadgerDB, error) { return b, nil } +// OpenTruncate opens (initializing and/or truncating if necessary) a Badger +// database at the specified path. Users must close the DB with Close(). +// +// Prefer Open in the general case, but after a bad shutdown it may be necessary +// to call OpenTruncate. This may cause data loss. Detect this situation by +// looking for badger.ErrTruncateNeeded. +func OpenTruncate(path string) (*BadgerDB, error) { + // DefaultOptions sets synchronous writes to true (maximum data integrity). + // TODO implement logging? + db, err := badger.Open(badger.DefaultOptions(path).WithLogger(nil).WithTruncate(true)) + if err != nil { + return nil, errors.Wrapf(err, "open badger with truncate %s", path) + } + + b := &BadgerDB{DB: db} + b.startBackgroundCompaction() + + return b, nil +} + // startBackgroundCompaction starts a background loop that will call the // compaction method on the database. Badger does not do this automatically, so // we need to be sure to do so here (or elsewhere).