From 45dbac4354c7852a42a7d28049bdd2e289425e83 Mon Sep 17 00:00:00 2001 From: Mike Arpaia Date: Fri, 12 Aug 2016 11:05:48 -0700 Subject: [PATCH] Using viper and cobra for config/commands (#67) --- README.md | 2 +- app/auth.go | 6 +- app/models.go | 4 +- app/osquery.go | 6 +- app/server.go | 8 +- circle.yml | 2 + config/config.go | 104 ----------- kolide.go | 314 +++++++++++++++++++++++----------- tools/app/example_config.json | 23 --- tools/app/kolide.yaml | 8 + 10 files changed, 233 insertions(+), 244 deletions(-) delete mode 100644 config/config.go delete mode 100644 tools/app/example_config.json create mode 100644 tools/app/kolide.yaml diff --git a/README.md b/README.md index 797b3ffac8..28ca940ec6 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Once you `docker-compose up` and are running the databases, you can build the code and run the following command to create the database tables: ``` -kolide prepare-db +kolide prepare db ``` ### Running Kolide diff --git a/app/auth.go b/app/auth.go index d261476e0f..e3ff46b226 100644 --- a/app/auth.go +++ b/app/auth.go @@ -8,8 +8,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/config" "github.com/kolide/kolide-ose/errors" + "github.com/spf13/viper" "golang.org/x/crypto/bcrypt" ) @@ -138,12 +138,12 @@ func generateRandomText(keySize int) (string, error) { func HashPassword(salt, password string) ([]byte, error) { return bcrypt.GenerateFromPassword( []byte(fmt.Sprintf("%s%s", password, salt)), - config.App.BcryptCost, + viper.GetInt("auth.bcrypt_cost"), ) } func SaltAndHashPassword(password string) (string, []byte, error) { - salt, err := generateRandomText(config.App.SaltKeySize) + salt, err := generateRandomText(viper.GetInt("auth.salt_key_size")) if err != nil { return "", []byte{}, err } diff --git a/app/models.go b/app/models.go index a8f533cb8a..c537cfd886 100644 --- a/app/models.go +++ b/app/models.go @@ -5,10 +5,10 @@ import ( "github.com/Sirupsen/logrus" "github.com/jinzhu/gorm" + "github.com/spf13/viper" _ "github.com/jinzhu/gorm/dialects/mysql" _ "github.com/jinzhu/gorm/dialects/sqlite" - "github.com/kolide/kolide-ose/config" "github.com/kolide/kolide-ose/sessions" ) @@ -34,7 +34,7 @@ func setDBSettings(db *gorm.DB) { // If debug mode is enabled, tell gorm to turn on logmode (log each // query as it is executed) - if config.App.Debug { + if viper.GetBool("debug") { db.LogMode(true) } } diff --git a/app/osquery.go b/app/osquery.go index fbecc9ce8d..dc35a55d75 100644 --- a/app/osquery.go +++ b/app/osquery.go @@ -8,8 +8,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/config" "github.com/kolide/kolide-ose/errors" + "github.com/spf13/viper" ) type ScheduledQuery struct { @@ -192,7 +192,7 @@ type OsqueryDistributedWritePostBody struct { // Generate a node key using NodeKeySize random bytes Base64 encoded func newNodeKey() (string, error) { - return generateRandomText(config.Osquery.NodeKeySize) + return generateRandomText(viper.GetInt("osquery.node_key_size")) } // Enroll a host. Even if this is an existing host, a new node key should be @@ -248,7 +248,7 @@ func OsqueryEnroll(c *gin.Context) { return } - if body.EnrollSecret != config.Osquery.EnrollSecret { + if body.EnrollSecret != viper.GetString("osquery.enroll_secret") { errors.ReturnError( c, errors.NewWithStatus(http.StatusUnauthorized, diff --git a/app/server.go b/app/server.go index 91772c93fa..ed6e0f9aea 100644 --- a/app/server.go +++ b/app/server.go @@ -13,9 +13,9 @@ import ( "github.com/gin-gonic/contrib/static" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" - "github.com/kolide/kolide-ose/config" "github.com/kolide/kolide-ose/errors" "github.com/kolide/kolide-ose/sessions" + "github.com/spf13/viper" "gopkg.in/go-playground/validator.v8" ) @@ -104,9 +104,9 @@ func CreateServer(db *gorm.DB, w io.Writer) *gin.Engine { sessions.Configure(&sessions.SessionConfiguration{ CookieName: "KolideSession", - JWTKey: config.App.JWTKey, - SessionKeySize: config.App.SessionKeySize, - Lifespan: config.App.SessionExpirationSeconds, + JWTKey: viper.GetString("auth.jwt_key"), + SessionKeySize: viper.GetInt("session.key_size"), + Lifespan: viper.GetFloat64("session.expiration_seconds"), }) // TODO: The following loggers are not synchronized with each other or diff --git a/circle.yml b/circle.yml index 197e5e4153..4e279f05ad 100644 --- a/circle.yml +++ b/circle.yml @@ -2,6 +2,8 @@ dependencies: pre: - make deps - go generate + override: + - go get -t -d -v ./... test: override: diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 398c887147..0000000000 --- a/config/config.go +++ /dev/null @@ -1,104 +0,0 @@ -package config - -import ( - "encoding/json" - "io/ioutil" -) - -type MySQLConfigData struct { - Address string `json:"address"` - Username string `json:"username"` - Password string `json:"password"` - Database string `json:"database"` -} - -type ServerConfigData struct { - Address string `json:"address"` - Cert string `json:"cert"` - Key string `json:"key"` -} - -type AppConfigData struct { - BcryptCost int `json:"bcrypt_cost"` - Debug bool `json:"debug"` - JWTKey string `json:"jwt_key"` - SaltKeySize int `json:"salt_key_size"` - SessionKeySize int `json:"session_key_size"` - SessionExpirationSeconds float64 `json:"session_expiration_seconds"` -} - -type OsqueryConfigData struct { - EnrollSecret string `json:"enroll_secret"` - NodeKeySize int `json:"node_key_size"` -} - -type configData struct { - MySQL MySQLConfigData `json:"mysql"` - Server ServerConfigData `json:"server"` - App AppConfigData `json:"app"` - Osquery OsqueryConfigData `json:"osquery"` -} - -var defaultMySQLConfigData = MySQLConfigData{ - Address: "mysql:3306", - Username: "kolide", - Password: "kolide", - Database: "kolide", -} - -var defaultServerConfigData = ServerConfigData{ - Address: "127.0.0.1:8080", - Cert: "./tools/osquery/kolide.crt", - Key: "./tools/osquery/kolide.key", -} - -var defaultAppConfigData = AppConfigData{ - BcryptCost: 12, - Debug: false, - JWTKey: "very secure", - SessionKeySize: 64, - SaltKeySize: 24, - SessionExpirationSeconds: 60 * 60 * 24 * 90, -} - -var defaultOsqueryConfigData = OsqueryConfigData{ - EnrollSecret: "bad secret", - NodeKeySize: 24, -} - -var defaultConfigData = configData{ - MySQL: defaultMySQLConfigData, - Server: defaultServerConfigData, - App: defaultAppConfigData, -} - -var ( - MySQL MySQLConfigData - Server ServerConfigData - App AppConfigData - Osquery OsqueryConfigData -) - -func init() { - MySQL = defaultMySQLConfigData - Server = defaultServerConfigData - App = defaultAppConfigData - Osquery = defaultOsqueryConfigData -} - -func LoadConfig(path string) error { - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } - var config configData - err = json.Unmarshal(content, &config) - if err != nil { - return err - } - MySQL = config.MySQL - App = config.App - Server = config.Server - Osquery = config.Osquery - return nil -} diff --git a/kolide.go b/kolide.go index 03682dd3f9..50e2f41635 100644 --- a/kolide.go +++ b/kolide.go @@ -8,125 +8,88 @@ import ( "os" "path" "runtime" + "strings" "time" "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" "github.com/kolide/kolide-ose/app" - "github.com/kolide/kolide-ose/config" - "gopkg.in/alecthomas/kingpin.v2" + "github.com/spf13/cobra" + "github.com/spf13/viper" ) var ( - appName = "kolide" - appDescription = "osquery command and control" - versionMajor = 0 - versionMinor = 1 - versionPatch = 0 - commitHash = "" - version = fmt.Sprintf("%d.%d.%d", versionMajor, versionMinor, versionPatch) - fullVersion = fmt.Sprintf("%d.%d.%d (commit: %v)", versionMajor, versionMinor, versionPatch, commitHash) + appName = "kolide" + versionMajor = 0 + versionMinor = 1 + versionPatch = 0 + version = fmt.Sprintf("%d.%d.%d", versionMajor, versionMinor, versionPatch) ) var ( - cli = kingpin.New(appName, appDescription) - - configPath = cli.Flag("config", "configuration file"). - Short('c'). - OverrideDefaultFromEnvar("KOLIDE_CONFIG_PATH"). - ExistingFile() - - debug = cli.Flag("debug", "Enable debug mode."). - OverrideDefaultFromEnvar("KOLIDE_DEBUG"). - Bool() - - logJson = cli.Flag("log_format_json", "Log in JSON format."). - OverrideDefaultFromEnvar("KOLIDE_LOG_FORMAT_JSON"). - Bool() - - prepareDB = cli.Command("prepare-db", "Create database tables") - serve = cli.Command("serve", "Run the Kolide server") + configFile string + debug bool ) -func init() { - // set gin mode to release to silence some superfluous logging - gin.SetMode(gin.ReleaseMode) +// RootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "kolide", + Short: "osquery management and orchestration", + Long: ` +osquery management and orchestration - // configure logging - logrus.AddHook(logContextHook{}) +Configurable Options: - rand.Seed(time.Now().UnixNano()) +Options may be supplied in a yaml configuration file or via environment +variables. You only need to define the configuration values for which you +wish to override the default value. + +Available Configurations: + + mysql: + address (string) (KOLIDE_MYSQL_ADDRESS) + username (string) (KOLIDE_MYSQL_USERNAME) + password (string) (KOLIDE_MYSQL_PASSWORD) + database (string) (KOLIDE_MYSQL_DATABASE) + server: + address (string) (KOLIDE_SERVER_ADDRESS) + cert (string) (KOLIDE_SERVER_CERT) + key (string) (KOLIDE_SERVER_KEY) + auth: + jwt_key (string) (KOLIDE_AUTH_JWT_KEY) + salt_key_size (int) (KOLIDE_AUTH_SALT_KEY_SIZE) + bcrypt_cost (int) (KOLIDE_AUTH_BCRYPT_COST) + session: + key_size (int) (KOLIDE_SESSION_KEY_SIZE) + expiration_seconds (float64) (KOLIDE_SESSION_EXPIRATION_SECONDS) + osquery: + enroll_secret (string) (KOLIDE_OSQUERY_ENROLL_SECRET) + node_key_size (int) (KOLIDE_OSQUERY_NODE_KEY_SIZE) +`, } -// logContextHook is a logrus hook which is used to contextualize application -// logs to include data stuch as line numbers, file names, etc. -type logContextHook struct{} +var serveCmd = &cobra.Command{ + Use: "serve", + Short: "Launch the kolide server", + Long: ` +Launch the kolide server -// Levels defines which levels the logContextHook logrus hook should apply to -func (hook logContextHook) Levels() []logrus.Level { - return logrus.AllLevels -} - -// Fire defines what the logContextHook should actually do when it is triggered -func (hook logContextHook) Fire(entry *logrus.Entry) error { - if pc, file, line, ok := runtime.Caller(8); ok { - funcName := runtime.FuncForPC(pc).Name() - - entry.Data["func"] = path.Base(funcName) - entry.Data["location"] = fmt.Sprintf("%s:%d", path.Base(file), line) - } - - return nil -} - -func main() { - // configure flag parsing and parse flags - cli.Version(version) - args, err := cli.Parse(os.Args[1:]) - - // configure the application based on the flags that have been set - if *debug { - config.App.Debug = true - logrus.SetLevel(logrus.DebugLevel) - } else { - logrus.SetLevel(logrus.WarnLevel) - } - - if *logJson { - logrus.SetFormatter(&logrus.JSONFormatter{}) - } - - // if config hasn't been defined and the example config exists relative to - // the binary, it's likely that the tool is being ran right after building - // from source so we auto-populate the example config path. - if *configPath == "" { - if _, err = os.Stat("./tools/app/example_config.json"); err == nil { - *configPath = "./tools/app/example_config.json" +Use kolide serve to run the main HTTPS server. The Kolide server bundles +together all static assets and dependent libraries into a statically linked go +binary (which you're executing right now). Use the options below to customize +the way that the kolide server works. +`, + Run: func(cmd *cobra.Command, args []string) { + if viper.Get("server.cert") == nil || viper.Get("server.key") == nil { + logrus.Fatal("TLS certificate and key were not found.") } - logrus.Warn("Using example config. These settings should be used for development only!") - } - // if the user has defined a config path OR the example config is found - // relative to the binary, load config content from the file. any content - // in the config file will overwrite the default values - if *configPath != "" { - err = config.LoadConfig(*configPath) - if err != nil { - logrus.Fatalf("Error loading config: %s", err.Error()) - } - } - - // route the executable based on the sub-command - switch kingpin.MustParse(args, err) { - case prepareDB.FullCommand(): - db, err := app.OpenDB(config.MySQL.Username, config.MySQL.Password, config.MySQL.Address, config.MySQL.Database) - if err != nil { - logrus.Fatalf("Error opening database: %s", err.Error()) - } - app.DropTables(db) - app.CreateTables(db) - case serve.FullCommand(): - db, err := app.OpenDB(config.MySQL.Username, config.MySQL.Password, config.MySQL.Address, config.MySQL.Database) + db, err := app.OpenDB( + viper.GetString("mysql.username"), + viper.GetString("mysql.password"), + viper.GetString("mysql.address"), + viper.GetString("mysql.database"), + ) if err != nil { logrus.Fatalf("Error opening database: %s", err.Error()) } @@ -150,18 +113,161 @@ $7777777....$....$777$.....+DI..DDD..DDI...8D...D8......$D:..8D....8D...8D...... ..... ...........I.................. . . . .. . . . . .. . . . . `) - fmt.Printf("=> %s %s application starting on https://%s\n", cli.Name, version, config.Server.Address) - fmt.Println("=> Run `kolide help serve` for more startup options") + fmt.Printf("=> Server starting on https://%s\n", viper.GetString("server.address")) + fmt.Println("=> Run `kolide serve --help` for more startup options") fmt.Println("Use Ctrl-C to stop") fmt.Print("\n\n") err = app.CreateServer(db, os.Stderr).RunTLS( - config.Server.Address, - config.Server.Cert, - config.Server.Key) + viper.GetString("server.address"), + viper.GetString("server.cert"), + viper.GetString("server.key"), + ) if err != nil { logrus.WithError(err).Fatal("Error running server") } + }, +} +var prepareCmd = &cobra.Command{ + Use: "prepare", + Short: "Subcommands for initializing kolide infrastructure", + Long: ` +Subcommands for initializing kolide infrastructure + +To setup kolide infrastructure, use one of the available commands. +`, + Run: func(cmd *cobra.Command, args []string) { + cmd.Help() + }, +} + +var dbCmd = &cobra.Command{ + Use: "db", + Short: "Given correct database configurations, prepare the databases for use", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + db, err := app.OpenDB( + viper.GetString("mysql.username"), + viper.GetString("mysql.password"), + viper.GetString("mysql.address"), + viper.GetString("mysql.database"), + ) + if err != nil { + logrus.Fatalf("Error opening database: %s", err.Error()) + } + app.DropTables(db) + app.CreateTables(db) + }, +} + +// Due to a deficiency in viper (https://github.com/spf13/viper/issues/71), one +// can not set the default values of nested config elements. For example, if the +// "mysql" section of the config allows a user to define "username", "password", +// and "database", but the only wants to override the default for "username". +// they should be able to create a config which looks like: +// +// mysql: +// username: foobar +// +// In viper, that would nullify the default values of all other config keys in +// the mysql section ("mysql.*"). To get around this, instead of using the +// provided API for setting default values, after we've read the config and env, +// we manually check to see if the value has been set and, if it hasn't, we set +// it manually. +func setDefaultConfigValue(key string, value interface{}) { + if viper.Get(key) == nil { + viper.Set(key, value) } } + +func initConfig() { + if configFile != "" { + viper.SetConfigFile(configFile) + } + + viper.SetConfigName("kolide") + viper.AddConfigPath(".") + viper.AddConfigPath("$HOME") + viper.AddConfigPath("./tools/app") + viper.AddConfigPath("/etc/kolide") + + viper.SetConfigType("yaml") + + viper.SetEnvPrefix("KOLIDE") + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AutomaticEnv() + + err := viper.ReadInConfig() + if err != nil { + logrus.Infoln("Not reading config file. Relying on environment variables and default values.") + } + + setDefaultConfigValue("mysql.address", "localhost:3306") + setDefaultConfigValue("mysql.username", "kolide") + setDefaultConfigValue("mysql.password", "kolide") + setDefaultConfigValue("mysql.database", "kolide") + + setDefaultConfigValue("server.address", "localhost:8080") + + setDefaultConfigValue("auth.bcrypt_cost", 12) + setDefaultConfigValue("auth.salt_key_size", 24) + + setDefaultConfigValue("session.key_size", 64) + setDefaultConfigValue("session.expiration_seconds", 60*60*24*90) + + setDefaultConfigValue("osquery.node_key_size", 24) + + if debug { + logrus.SetLevel(logrus.DebugLevel) + viper.Set("debug", true) + } else { + logrus.SetLevel(logrus.WarnLevel) + } + + if viper.GetBool("logs.json") { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } +} + +// logContextHook is a logrus hook which is used to contextualize application +// logs to include data stuch as line numbers, file names, etc. +type logContextHook struct{} + +// Levels defines which levels the logContextHook logrus hook should apply to +func (hook logContextHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +// Fire defines what the logContextHook should actually do when it is triggered +func (hook logContextHook) Fire(entry *logrus.Entry) error { + if pc, file, line, ok := runtime.Caller(8); ok { + funcName := runtime.FuncForPC(pc).Name() + + entry.Data["func"] = path.Base(funcName) + entry.Data["location"] = fmt.Sprintf("%s:%d", path.Base(file), line) + } + + return nil +} + +func init() { + gin.SetMode(gin.ReleaseMode) + + logrus.AddHook(logContextHook{}) + + rand.Seed(time.Now().UnixNano()) + + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&configFile, "config", "", "Path to a configuration file") + rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug logging and behavior") + + rootCmd.AddCommand(serveCmd) + rootCmd.AddCommand(prepareCmd) + prepareCmd.AddCommand(dbCmd) +} + +func main() { + rootCmd.Execute() +} diff --git a/tools/app/example_config.json b/tools/app/example_config.json deleted file mode 100644 index 6a676cafae..0000000000 --- a/tools/app/example_config.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "mysql": { - "address": "192.168.99.100:3306", - "username": "kolide", - "password": "kolide", - "database": "kolide" - }, - "server": { - "address": ":8080", - "cert": "./tools/osquery/kolide.crt", - "key": "./tools/osquery/kolide.key" - }, - "app": { - "bcrypt_cost": 12, - "salt_key_size": 12, - "jwt_key": "very secure", - "session_key_size": 64 - }, - "osquery": { - "enroll_secret": "super secure", - "node_key_size": 24 - } -} diff --git a/tools/app/kolide.yaml b/tools/app/kolide.yaml new file mode 100644 index 0000000000..797deb0ef3 --- /dev/null +++ b/tools/app/kolide.yaml @@ -0,0 +1,8 @@ +server: + cert: "./tools/osquery/kolide.crt" + key: "./tools/osquery/kolide.key" +auth: + jwt_key: very secure +osquery: + enroll_secret: super secure +debug: true \ No newline at end of file