diff --git a/.gitignore b/.gitignore index 37e62bb1d2..778cd232af 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ node_modules app/bindata.go *.cover *.test +*.log # operating system artifacts .DS_Store \ No newline at end of file diff --git a/app/email_test.go b/app/email_test.go index 6504050c2e..a9065b4638 100644 --- a/app/email_test.go +++ b/app/email_test.go @@ -3,6 +3,7 @@ package app import ( "bytes" "encoding/json" + "io/ioutil" "net/http" "net/http/httptest" "strings" @@ -60,7 +61,7 @@ func (w *mockResponseWriter) WriteHeader(int) { func TestUnauthenticatedPasswordReset(t *testing.T) { db := openTestDB(t) pool := newMockSMTPConnectionPool() - r := CreateServer(db, pool, &testLogger{t: t}) + r := CreateServer(db, pool, &testLogger{t: t}, &OsqueryLogWriter{Writer: ioutil.Discard}, &OsqueryLogWriter{Writer: ioutil.Discard}) admin, _ := NewUser(db, "admin", "foobar", "admin@kolide.co", true, false) { @@ -119,7 +120,7 @@ func TestUnauthenticatedPasswordReset(t *testing.T) { func TestAuthenticatedPasswordReset(t *testing.T) { db := openTestDB(t) pool := newMockSMTPConnectionPool() - r := CreateServer(db, pool, &testLogger{t: t}) + r := CreateServer(db, pool, &testLogger{t: t}, &OsqueryLogWriter{Writer: ioutil.Discard}, &OsqueryLogWriter{Writer: ioutil.Discard}) admin, _ := NewUser(db, "admin", "foobar", "admin@kolide.co", true, false) request, _ := http.NewRequest("GET", "/", nil) writer := newMockResponseWriter() diff --git a/app/osquery.go b/app/osquery.go index dc35a55d75..682c32ab27 100644 --- a/app/osquery.go +++ b/app/osquery.go @@ -1,7 +1,9 @@ package app import ( + "encoding/json" "fmt" + "io" "net/http" "time" @@ -12,6 +14,51 @@ import ( "github.com/spf13/viper" ) +// The "output plugin" interface for osquery result logs. The implementer of +// this interface can do whatever processing they would like with the log, +// returning the appropriate error status +type OsqueryResultHandler interface { + HandleResultLog(log OsqueryResultLog, nodeKey string) error +} + +// The "output plugin" interface for osquery status logs. The implementer of +// this interface can do whatever processing they would like with the log, +// returning the appropriate error status +type OsqueryStatusHandler interface { + HandleStatusLog(log OsqueryStatusLog, nodeKey string) error +} + +// This struct is used for injecting dependencies for osquery TLS processing. +// It can be configured in a `main` function to bind the appropriate handlers +// and it's methods can be attached to routes. +type OsqueryHandler struct { + ResultHandler OsqueryResultHandler + StatusHandler OsqueryStatusHandler +} + +// Basic implementation of the `OsqueryResultHandler` and +// `OsqueryStatusHandler` interfaces. It will write the logs to the io.Writer +// provided in Writer. +type OsqueryLogWriter struct { + Writer io.Writer +} + +func (w *OsqueryLogWriter) HandleStatusLog(log OsqueryStatusLog, nodeKey string) error { + err := json.NewEncoder(w.Writer).Encode(log) + if err != nil { + return errors.NewFromError(err, http.StatusInternalServerError, "error writing result log") + } + return nil +} + +func (w *OsqueryLogWriter) HandleResultLog(log OsqueryResultLog, nodeKey string) error { + err := json.NewEncoder(w.Writer).Encode(log) + if err != nil { + return errors.NewFromError(err, http.StatusInternalServerError, "error writing status log") + } + return nil +} + type ScheduledQuery struct { ID uint `gorm:"primary_key"` CreatedAt time.Time @@ -159,26 +206,27 @@ type OsqueryConfigPostBody struct { } type OsqueryLogPostBody struct { - NodeKey string `json:"node_key" validate:"required"` - LogType string `json:"log_type" validate:"required"` - Data []map[string]interface{} `json:"data" validate:"required"` + NodeKey string `json:"node_key" validate:"required"` + LogType string `json:"log_type" validate:"required"` + Data *json.RawMessage `json:"data" validate:"required"` } type OsqueryResultLog struct { - Name string `json:"name"` - HostIdentifier string `json:"hostIdentifier"` - UnixTime string `json:"unixTime"` - CalendarTime string `json:"calendarTime"` + Name string `json:"name" validate:"required"` + HostIdentifier string `json:"hostIdentifier" validate:"required"` + UnixTime string `json:"unixTime" validate:"required"` + CalendarTime string `json:"calendarTime" validate:"required"` Columns map[string]string `json:"columns"` - Action string `json:"action"` + Action string `json:"action" validate:"required"` } type OsqueryStatusLog struct { - Severity string `json:"severity"` - Filename string `json:"filename"` - Line string `json:"line"` - Message string `json:"message"` - Version string `json:"version"` + Severity string `json:"severity" validate:"required"` + Filename string `json:"filename" validate:"required"` + Line string `json:"line" validate:"required"` + Message string `json:"message" validate:"required"` + Version string `json:"version" validate:"required"` + Decorations map[string]string `json:"decorations"` } type OsqueryDistributedReadPostBody struct { @@ -244,12 +292,12 @@ func OsqueryEnroll(c *gin.Context) { var body OsqueryEnrollPostBody err := ParseAndValidateJSON(c, &body) if err != nil { - errors.ReturnError(c, err) + errors.ReturnOsqueryError(c, err) return } if body.EnrollSecret != viper.GetString("osquery.enroll_secret") { - errors.ReturnError( + errors.ReturnOsqueryError( c, errors.NewWithStatus(http.StatusUnauthorized, "Node key invalid", @@ -263,7 +311,7 @@ func OsqueryEnroll(c *gin.Context) { host, err := EnrollHost(db, body.HostIdentifier, "", "", "") if err != nil { - errors.ReturnError(c, errors.DatabaseError(err)) + errors.ReturnOsqueryError(c, errors.DatabaseError(err)) return } @@ -280,7 +328,7 @@ func OsqueryConfig(c *gin.Context) { var body OsqueryConfigPostBody err := ParseAndValidateJSON(c, &body) if err != nil { - errors.ReturnError(c, err) + errors.ReturnOsqueryError(c, err) return } logrus.Debugf("OsqueryConfig: %s", body.NodeKey) @@ -297,77 +345,130 @@ func OsqueryConfig(c *gin.Context) { }) } -func OsqueryLog(c *gin.Context) { +// Authenticate a (post-enrollment) TLS request from osqueryd. To do this we +// verify that the provided node key is valid. +func authenticateRequest(db *gorm.DB, nodeKey string) error { + host := Host{NodeKey: nodeKey} + err := db.Where(&host).First(&host).Error + if err != nil { + switch err { + case gorm.ErrRecordNotFound: + e := errors.NewFromError( + err, + http.StatusUnauthorized, + "Unauthorized", + ) + // osqueryd expects the literal string "true" here + e.Extra = map[string]interface{}{"node_invalid": "true"} + return e + default: + return errors.DatabaseError(err) + } + } + + return nil +} + +// Unmarshal the status logs before sending them to the status log handler +func (h *OsqueryHandler) handleStatusLogs(db *gorm.DB, data *json.RawMessage, nodeKey string) error { + var statuses []OsqueryStatusLog + if err := json.Unmarshal(*data, &statuses); err != nil { + return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error") + } + // Perhaps we should validate the unmarshalled status log + + for _, status := range statuses { + if err := h.StatusHandler.HandleStatusLog(status, nodeKey); err != nil { + return err + } + logrus.Debugf("Osquery status: %+v", status) + } + + return nil +} + +// Unmarshal the result logs before sending them to the result log handler +func (h *OsqueryHandler) handleResultLogs(db *gorm.DB, data *json.RawMessage, nodeKey string) error { + var results []OsqueryResultLog + if err := json.Unmarshal(*data, &results); err != nil { + return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error") + } + // Perhaps we should validate the unmarshalled result log + + for _, result := range results { + if err := h.ResultHandler.HandleResultLog(result, nodeKey); err != nil { + return err + } + logrus.Debugf("Osquery result: %+v", result) + } + + return nil +} + +// Set the update time for the provided host to indicate that it has +// successfully checked in. +func updateLastSeen(db *gorm.DB, host *Host) error { + updateTime := time.Now() + err := db.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", updateTime, host.NodeKey).Error + if err != nil { + return errors.DatabaseError(err) + } + host.UpdatedAt = updateTime + return nil +} + +// Endpoint used by the osqueryd TLS logger plugin +func (h *OsqueryHandler) OsqueryLog(c *gin.Context) { var body OsqueryLogPostBody err := ParseAndValidateJSON(c, &body) if err != nil { - errors.ReturnError(c, err) + errors.ReturnOsqueryError(c, err) return } - logrus.Debugf("OsqueryLog: %s", body.LogType) - if body.LogType == "status" { - for _, data := range body.Data { - var log OsqueryStatusLog + db := GetDB(c) - severity, ok := data["severity"].(string) - if ok { - log.Severity = severity - } else { - logrus.Error("Error asserting the type of status log severity") - } - - filename, ok := data["filename"].(string) - if ok { - log.Filename = filename - } else { - logrus.Error("Error asserting the type of status log filename") - } - - line, ok := data["line"].(string) - if ok { - log.Line = line - } else { - logrus.Error("Error asserting the type of status log line") - } - - message, ok := data["message"].(string) - if ok { - log.Message = message - } else { - logrus.Error("Error asserting the type of status log message") - } - - version, ok := data["version"].(string) - if ok { - log.Version = version - } else { - logrus.Error("Error asserting the type of status log version") - } - - logrus.WithFields(logrus.Fields{ - "node_key": body.NodeKey, - "severity": log.Severity, - "filename": log.Filename, - "line": log.Line, - "version": log.Version, - }).Info(log.Message) - } - } else if body.LogType == "result" { - // TODO: handle all of the different kinds of results logs + err = authenticateRequest(db, body.NodeKey) + if err != nil { + errors.ReturnOsqueryError(c, err) + return } - c.JSON(http.StatusOK, - gin.H{ - "node_invalid": false, - }) + switch body.LogType { + case "status": + err = h.handleStatusLogs(db, body.Data, body.NodeKey) + + case "result": + err = h.handleResultLogs(db, body.Data, body.NodeKey) + + default: + err = errors.NewWithStatus( + errors.StatusUnprocessableEntity, + "Unknown result type", + fmt.Sprintf("Unknown result type: %s", body.LogType), + ) + } + + if err != nil { + errors.ReturnOsqueryError(c, err) + return + } + + err = updateLastSeen(db, &Host{NodeKey: body.NodeKey}) + + if err != nil { + errors.ReturnOsqueryError(c, err) + return + } + + c.JSON(http.StatusOK, gin.H{}) } func OsqueryDistributedRead(c *gin.Context) { var body OsqueryDistributedReadPostBody err := ParseAndValidateJSON(c, &body) if err != nil { - errors.ReturnError(c, err) + errors.ReturnOsqueryError(c, err) return } logrus.Debugf("OsqueryDistributedRead: %s", body.NodeKey) @@ -385,7 +486,7 @@ func OsqueryDistributedWrite(c *gin.Context) { var body OsqueryDistributedWritePostBody err := ParseAndValidateJSON(c, &body) if err != nil { - errors.ReturnError(c, err) + errors.ReturnOsqueryError(c, err) return } logrus.Debugf("OsqueryDistributedWrite: %s", body.NodeKey) diff --git a/app/osquery_integration_test.go b/app/osquery_integration_test.go index 2117696949..e35c83fb25 100644 --- a/app/osquery_integration_test.go +++ b/app/osquery_integration_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIntegrationEnrollHostBadSecret(t *testing.T) { +func TestOsqueryIntegrationEnrollHostBadSecret(t *testing.T) { var req IntegrationRequests req.New(t) @@ -33,7 +33,7 @@ func TestIntegrationEnrollHostBadSecret(t *testing.T) { } } -func TestIntegrationEnrollHostMissingIdentifier(t *testing.T) { +func TestOsqueryIntegrationEnrollHostMissingIdentifier(t *testing.T) { var req IntegrationRequests req.New(t) @@ -52,10 +52,10 @@ func TestIntegrationEnrollHostMissingIdentifier(t *testing.T) { t.Fatalf("JSON decode error: %s JSON contents:\n %s", err.Error(), resp.Body.Bytes()) } - assert.Equal(t, "Validation error", body["message"]) + assert.Equal(t, "Validation error", body["error"]) } -func TestIntegrationEnrollHostGood(t *testing.T) { +func TestOsqueryIntegrationEnrollHostGood(t *testing.T) { var req IntegrationRequests req.New(t) @@ -134,3 +134,104 @@ func TestIntegrationEnrollHostGood(t *testing.T) { } } + +func TestOsqueryIntegrationOsqueryLogErrors(t *testing.T) { + var req IntegrationRequests + req.New(t) + + // Missing log type should give error + data := json.RawMessage("{}") + resp := req.OsqueryLog("node_key", "", &data) + + assert.Equal(t, errors.StatusUnprocessableEntity, resp.Code) + + body := map[string]interface{}{} + assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body)) + assert.Contains(t, body, "error") + + // Bad node key should give error + resp = req.OsqueryLog("bad_node_key", "status", &data) + + assert.Equal(t, http.StatusUnauthorized, resp.Code) + + body = map[string]interface{}{} + assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body)) + assert.Contains(t, body, "error") + assert.Contains(t, body, "node_invalid") + assert.Equal(t, "true", body["node_invalid"]) +} + +func TestOsqueryIntegrationOsqueryLogSuccess(t *testing.T) { + var req IntegrationRequests + req.New(t) + + // First enroll + resp := req.EnrollHost("super secret", "fake_host_1") + + body := map[string]interface{}{} + assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body)) + assert.Equal(t, http.StatusOK, resp.Code) + assert.Contains(t, body, "node_key") + nodeKey := body["node_key"].(string) + + // Now logging should be successful + + expectStatus := []OsqueryStatusLog{ + OsqueryStatusLog{ + Severity: "bad", + Filename: "nope.cpp", + Line: "42", + Message: "bad stuff happened", + Version: "1.8.0", + Decorations: map[string]string{ + "foo": "bar", + }, + }, + OsqueryStatusLog{ + Severity: "worse", + Filename: "uhoh.cpp", + Line: "42", + Message: "bad stuff happened", + Version: "1.8.0", + Decorations: map[string]string{ + "foo": "bar", + "baz": "bang", + }, + }, + } + + expectResult := []OsqueryResultLog{ + OsqueryResultLog{ + Name: "query", + HostIdentifier: "somehost", + UnixTime: "the time", + CalendarTime: "other time", + Columns: map[string]string{ + "foo": "bar", + "baz": "bang", + }, + Action: "none", + }, + } + + // Send status log + jsonVal, err := json.Marshal(&expectStatus) + assert.NoError(t, err) + data := json.RawMessage(jsonVal) + + resp = req.OsqueryLog(nodeKey, "status", &data) + assert.Equal(t, http.StatusOK, resp.Code) + + // Send result log + jsonVal, err = json.Marshal(&expectResult) + assert.NoError(t, err) + data = json.RawMessage(jsonVal) + + resp = req.OsqueryLog(nodeKey, "result", &data) + assert.Equal(t, http.StatusOK, resp.Code) + + // Check that correct logs were logged + assert.Equal(t, expectStatus, req.statusHandler.Logs) + assert.Equal(t, expectResult, req.resultHandler.Logs) + +} diff --git a/app/osquery_test.go b/app/osquery_test.go index a44e7b42ab..bbb87f3dc7 100644 --- a/app/osquery_test.go +++ b/app/osquery_test.go @@ -1,7 +1,12 @@ package app import ( + "bytes" + "encoding/json" "testing" + "time" + + "github.com/stretchr/testify/assert" ) func TestEnrollHost(t *testing.T) { @@ -91,3 +96,177 @@ func TestReEnrollHost(t *testing.T) { } } + +func TestOsqueryLogWriterStatus(t *testing.T) { + buf := bytes.Buffer{} + logWriter := OsqueryLogWriter{Writer: &buf} + log := OsqueryStatusLog{ + Severity: "bad", + Filename: "nope.cpp", + Line: "42", + Message: "bad stuff happened", + Version: "1.8.0", + Decorations: map[string]string{ + "foo": "bar", + }, + } + + assert.NoError(t, logWriter.HandleStatusLog(log, "foo")) + + jsonStr, err := json.Marshal(&log) + assert.NoError(t, err) + assert.Equal(t, string(jsonStr)+"\n", buf.String()) + +} + +func TestOsqueryLogWriterResult(t *testing.T) { + buf := bytes.Buffer{} + logWriter := OsqueryLogWriter{Writer: &buf} + log := OsqueryResultLog{ + Name: "query", + HostIdentifier: "somehost", + UnixTime: "the time", + CalendarTime: "other time", + Columns: map[string]string{ + "foo": "bar", + "baz": "bang", + }, + Action: "none", + } + + assert.NoError(t, logWriter.HandleResultLog(log, "foo")) + + jsonStr, err := json.Marshal(&log) + assert.NoError(t, err) + assert.Equal(t, string(jsonStr)+"\n", buf.String()) + +} + +func TestOsqueryHandlerHandleStatusLogs(t *testing.T) { + writer := new(mockOsqueryStatusLogWriter) + handler := OsqueryHandler{StatusHandler: writer} + + data := json.RawMessage("{") + assert.Error(t, + handler.handleStatusLogs(nil, &data, "foo"), + "should error with bad json", + ) + + expect := []OsqueryStatusLog{ + OsqueryStatusLog{ + Severity: "bad", + Filename: "nope.cpp", + Line: "42", + Message: "bad stuff happened", + Version: "1.8.0", + Decorations: map[string]string{ + "foo": "bar", + }, + }, + OsqueryStatusLog{ + Severity: "worse", + Filename: "uhoh.cpp", + Line: "42", + Message: "bad stuff happened", + Version: "1.8.0", + Decorations: map[string]string{ + "foo": "bar", + "baz": "bang", + }, + }, + } + + jsonVal, err := json.Marshal(&expect) + assert.NoError(t, err) + data = json.RawMessage(jsonVal) + assert.NoError(t, handler.handleStatusLogs(nil, &data, "foo")) + + assert.Equal(t, expect, writer.Logs) +} + +func TestOsqueryHandlerHandleResultLogs(t *testing.T) { + writer := new(mockOsqueryResultLogWriter) + handler := OsqueryHandler{ResultHandler: writer} + + data := json.RawMessage("{") + assert.Error(t, + handler.handleResultLogs(nil, &data, "foo"), + "should error with bad json", + ) + + expect := []OsqueryResultLog{ + OsqueryResultLog{ + Name: "query", + HostIdentifier: "somehost", + UnixTime: "the time", + CalendarTime: "other time", + Columns: map[string]string{ + "foo": "bar", + "baz": "bang", + }, + Action: "none", + }, + OsqueryResultLog{ + Name: "other query", + HostIdentifier: "somehost", + UnixTime: "the time", + CalendarTime: "other time", + Columns: map[string]string{ + "jim": "jam", + "foo": "bar", + "baz": "bang", + }, + Action: "none", + }, + } + + jsonVal, err := json.Marshal(&expect) + assert.NoError(t, err) + + data = json.RawMessage(jsonVal) + assert.NoError(t, handler.handleResultLogs(nil, &data, "foo")) + + assert.Equal(t, expect, writer.Logs) +} + +func TestOsqueryAuthenticateRequest(t *testing.T) { + db := openTestDB(t) + + assert.Error(t, authenticateRequest(db, "foo"), "bad node key should fail") + + host, err := EnrollHost(db, "fake_uuid", "fake_hostname", "", "") + + assert.NoError(t, err, "enroll should succeed") + assert.NoError(t, authenticateRequest(db, host.NodeKey), "enrolled host should pass auth") + + // re-enroll so the old node key is no longer valid + oldNodeKey := host.NodeKey + host, err = EnrollHost(db, "fake_uuid", "fake_hostname", "", "") + + assert.NoError(t, err, "re-enroll should succeed") + // auth should fail now + assert.Error(t, authenticateRequest(db, oldNodeKey), "auth should succeed") +} + +func TestOsqueryUpdateLastSeen(t *testing.T) { + db := openTestDB(t) + + host, err := EnrollHost(db, "fake_uuid", "fake_hostname", "", "") + assert.NoError(t, err) + + // Set update time to 0, then call update and make sure it is updated + assert.NoError(t, db.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", time.Time{}, host.NodeKey).Error) + + // Clear and reload host + host = &Host{NodeKey: host.NodeKey} + assert.NoError(t, db.Where(&host).First(&host).Error) + assert.True(t, host.UpdatedAt.IsZero()) + + assert.NoError(t, updateLastSeen(db, host)) + assert.False(t, host.UpdatedAt.IsZero()) + + // Make sure the update propagated to the DB + host = &Host{NodeKey: host.NodeKey} + assert.NoError(t, db.Where(&host).First(&host).Error) + assert.False(t, host.UpdatedAt.IsZero()) +} diff --git a/app/server.go b/app/server.go index 670544b938..d6187febdd 100644 --- a/app/server.go +++ b/app/server.go @@ -93,7 +93,7 @@ func NewSessionManager(c *gin.Context) *sessions.SessionManager { func parseJSON(c *gin.Context, obj interface{}) error { decoder := json.NewDecoder(c.Request.Body) if err := decoder.Decode(obj); err != nil { - return err + return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error") } return nil } @@ -102,7 +102,7 @@ func parseJSON(c *gin.Context, obj interface{}) error { // the validator library. func ParseAndValidateJSON(c *gin.Context, obj interface{}) error { if err := parseJSON(c, obj); err != nil { - return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error") + return err } return validate.Struct(obj) @@ -120,7 +120,7 @@ func NotFound(c *gin.Context) { // CreateServer creates a gin.Engine HTTP server and configures it to be in a // state such that it is ready to serve HTTP requests for the kolide application -func CreateServer(db *gorm.DB, pool SMTPConnectionPool, w io.Writer) *gin.Engine { +func CreateServer(db *gorm.DB, pool SMTPConnectionPool, w io.Writer, resultHandler OsqueryResultHandler, statusHandler OsqueryStatusHandler) *gin.Engine { server := gin.New() server.Use(DatabaseMiddleware(db)) server.Use(SMTPConnectionPoolMiddleware(pool)) @@ -189,9 +189,15 @@ func CreateServer(db *gorm.DB, pool SMTPConnectionPool, w io.Writer) *gin.Engine // osquery API endpoints osq := v1.Group("/osquery") + + osqueryHandler := OsqueryHandler{ + ResultHandler: resultHandler, + StatusHandler: statusHandler, + } + osq.POST("/enroll", OsqueryEnroll) osq.POST("/config", OsqueryConfig) - osq.POST("/log", OsqueryLog) + osq.POST("/log", osqueryHandler.OsqueryLog) osq.POST("/distributed/read", OsqueryDistributedRead) osq.POST("/distributed/write", OsqueryDistributedWrite) diff --git a/app/test_util.go b/app/test_util.go index 0fe2fba599..48e2498315 100644 --- a/app/test_util.go +++ b/app/test_util.go @@ -43,11 +43,31 @@ func openTestDB(t *testing.T) *gorm.DB { return db } +type mockOsqueryStatusLogWriter struct { + Logs []OsqueryStatusLog +} + +func (w *mockOsqueryStatusLogWriter) HandleStatusLog(log OsqueryStatusLog, nodeKey string) error { + w.Logs = append(w.Logs, log) + return nil +} + +type mockOsqueryResultLogWriter struct { + Logs []OsqueryResultLog +} + +func (w *mockOsqueryResultLogWriter) HandleResultLog(log OsqueryResultLog, nodeKey string) error { + w.Logs = append(w.Logs, log) + return nil +} + type IntegrationRequests struct { - r *gin.Engine - db *gorm.DB - pool SMTPConnectionPool - t *testing.T + r *gin.Engine + db *gorm.DB + pool SMTPConnectionPool + t *testing.T + statusHandler *mockOsqueryStatusLogWriter + resultHandler *mockOsqueryResultLogWriter } func (req *IntegrationRequests) New(t *testing.T) { @@ -56,6 +76,9 @@ func (req *IntegrationRequests) New(t *testing.T) { req.db = openTestDB(t) req.pool = newMockSMTPConnectionPool() + req.statusHandler = new(mockOsqueryStatusLogWriter) + req.resultHandler = new(mockOsqueryResultLogWriter) + // Until we have a better solution for first-user onboarding, manually // create an admin _, err := NewUser(req.db, "admin", "foobar", "admin@kolide.co", true, false) @@ -63,7 +86,8 @@ func (req *IntegrationRequests) New(t *testing.T) { t.Fatalf("Error opening DB: %s", err.Error()) } - req.r = CreateServer(req.db, req.pool, &testLogger{t: t}) + gin.SetMode(gin.ReleaseMode) + req.r = CreateServer(req.db, req.pool, &testLogger{t: t}, req.resultHandler, req.statusHandler) } func (req *IntegrationRequests) Login(username, password string, sessionOut *string) { @@ -476,3 +500,26 @@ func (req *IntegrationRequests) EnrollHost(enrollSecret, hostIdentifier string) return response } + +func (req *IntegrationRequests) OsqueryLog(nodeKey, logType string, data *json.RawMessage) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + body, err := json.Marshal(OsqueryLogPostBody{ + NodeKey: nodeKey, + LogType: logType, + Data: data, + }) + + if err != nil { + req.t.Fatal(err.Error()) + } + + buff := new(bytes.Buffer) + buff.Write(body) + req.t.Log(buff.String()) + + request, _ := http.NewRequest("POST", "/api/v1/osquery/log", buff) + request.Header.Set("Content-Type", "application/json") + req.r.ServeHTTP(response, request) + + return response +} diff --git a/errors/errors.go b/errors/errors.go index 5326aaec99..92f3394e92 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -12,12 +12,15 @@ import ( // Kolide's internal representation for errors. It can be used to wrap another // error (stored in Err), and additionally contains fields for public // (PublicMessage) and private (PrivateMessage) error messages as well as the -// HTTP status code (StatusCode) corresponding to the error. +// HTTP status code (StatusCode) corresponding to the error. Extra holds extra +// information that will be inserted as top level key/value pairs in the error +// response. type KolideError struct { Err error StatusCode int PublicMessage string PrivateMessage string + Extra map[string]interface{} } // Implementation of error interface @@ -73,11 +76,25 @@ const StatusUnprocessableEntity = 422 // Handle an error, printing debug information, writing to the HTTP response as // appropriate for the dynamic error type. func ReturnError(c *gin.Context, err error) { + baseReturnError(c, err, "message") +} + +// osqueryd does not check the HTTP status code, and only looks for a +func ReturnOsqueryError(c *gin.Context, err error) { + baseReturnError(c, err, "error") +} + +func baseReturnError(c *gin.Context, err error, messageKey string) { switch typedErr := err.(type) { case *KolideError: - c.JSON(typedErr.StatusCode, - gin.H{"message": typedErr.PublicMessage}) + errJSON := map[string]interface{}{} + for key, val := range typedErr.Extra { + errJSON[key] = val + } + errJSON[messageKey] = typedErr.PublicMessage + + c.JSON(typedErr.StatusCode, errJSON) logrus.WithError(typedErr.Err).Debug(typedErr.PrivateMessage) case validator.ValidationErrors: @@ -91,19 +108,19 @@ func ReturnError(c *gin.Context, err error) { } c.JSON(StatusUnprocessableEntity, - gin.H{"message": "Validation error", + gin.H{messageKey: "Validation error", "errors": errors, }) logrus.WithError(typedErr).Debug("Validation error") case gorm.Errors, *gorm.Errors: c.JSON(http.StatusInternalServerError, - gin.H{"message": "Database error"}) + gin.H{messageKey: "Database error"}) logrus.WithError(typedErr).Debug(typedErr.Error()) default: c.JSON(http.StatusInternalServerError, - gin.H{"message": "Unspecified error"}) + gin.H{messageKey: "Unspecified error"}) logrus.WithError(typedErr).Debug("Unspecified error") } } diff --git a/glide.lock b/glide.lock index f4863fc66c..3dd8e4ab11 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: cbd4502309ad1575151e0f9734937a85056213c326389ecc350259e677f2bfb6 -updated: 2016-08-13T17:43:32.844377583-04:00 +hash: 4ffa52b45a47295720fcf4dfcc0cc85c9f47a35a3bd43bdc6e0c2355bdd76109 +updated: 2016-08-17T10:01:54.299187864-07:00 imports: - name: github.com/alecthomas/template version: a0175ee3bccc567396460bf5acd36800cb10c49c @@ -123,6 +123,8 @@ imports: version: e5900212cbf65b181d3d8e08308ef06a01d117cf - name: gopkg.in/go-playground/validator.v8 version: 5f57d2222ad794d0dffb07e664ea05e2ee07d60c +- name: gopkg.in/natefinch/lumberjack.v2 + version: 514cbda263a734ae8caac038dadf05f8f3f9f738 - name: gopkg.in/yaml.v2 version: e4d366fc3c7938e2958e662b4258c7a89e1f0e3e testImports: [] diff --git a/glide.yaml b/glide.yaml index 228f291bc9..6a22e3820c 100644 --- a/glide.yaml +++ b/glide.yaml @@ -74,3 +74,5 @@ import: version: fd703108daeb23d77c124d12978e9b6c4f28f034 - package: github.com/spf13/cobra - package: github.com/spf13/viper +- package: gopkg.in/natefinch/lumberjack.v2 + version: v2.0 diff --git a/kolide.go b/kolide.go index 8d745f3c31..241a02c93f 100644 --- a/kolide.go +++ b/kolide.go @@ -19,6 +19,7 @@ import ( "github.com/kolide/kolide-ose/app" "github.com/spf13/cobra" "github.com/spf13/viper" + "gopkg.in/natefinch/lumberjack.v2" ) var ( @@ -43,8 +44,8 @@ osquery management and orchestration Configurable Options: -Options may be supplied in a yaml configuration file or via environment -variables. You only need to define the configuration values for which you +Options may be supplied in a yaml configuration file or via environment +variables. You only need to define the configuration values for which you wish to override the default value. Available Configurations: @@ -138,7 +139,33 @@ $7777777....$....$777$.....+DI..DDD..DDI...8D...D8......$D:..8D....8D...8D...... fmt.Println("Use Ctrl-C to stop") fmt.Print("\n\n") - err = app.CreateServer(db, smtpConnectionPool, os.Stderr).RunTLS( + resultFile := viper.GetString("osquery.result_log_file") + resultHandler := &app.OsqueryLogWriter{ + Writer: &lumberjack.Logger{ + Filename: resultFile, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + }, + } + + statusFile := viper.GetString("osquery.status_log_file") + statusHandler := &app.OsqueryLogWriter{ + Writer: &lumberjack.Logger{ + Filename: statusFile, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + }, + } + + err = app.CreateServer( + db, + smtpConnectionPool, + os.Stderr, + resultHandler, + statusHandler, + ).RunTLS( viper.GetString("server.address"), viper.GetString("server.cert"), viper.GetString("server.key"), @@ -154,7 +181,7 @@ var prepareCmd = &cobra.Command{ Short: "Subcommands for initializing kolide infrastructure", Long: ` Subcommands for initializing kolide infrastructure - + To setup kolide infrastructure, use one of the available commands. `, Run: func(cmd *cobra.Command, args []string) { @@ -228,9 +255,9 @@ func initConfig() { setDefaultConfigValue("mysql.password", "kolide") setDefaultConfigValue("mysql.database", "kolide") - setDefaultConfigValue("server.address", "localhost:8080") + setDefaultConfigValue("server.address", "0.0.0.0:8080") - setDefaultConfigValue("app.web_address", "localhost:8080") + setDefaultConfigValue("app.web_address", "0.0.0.0:8080") setDefaultConfigValue("auth.bcrypt_cost", 12) setDefaultConfigValue("auth.salt_key_size", 24) @@ -243,6 +270,8 @@ func initConfig() { setDefaultConfigValue("session.expiration_seconds", 60*60*24*90) setDefaultConfigValue("osquery.node_key_size", 24) + setDefaultConfigValue("osquery.status_log_file", "/tmp/osquery_status") + setDefaultConfigValue("osquery.result_log_file", "/tmp/osquery_result") if debug { logrus.SetLevel(logrus.DebugLevel) diff --git a/tools/app/kolide.yaml b/tools/app/kolide.yaml index 797deb0ef3..f361159fda 100644 --- a/tools/app/kolide.yaml +++ b/tools/app/kolide.yaml @@ -5,4 +5,7 @@ auth: jwt_key: very secure osquery: enroll_secret: super secure + result_log_file: /tmp/osquery_result + status_log_file: /tmp/osquery_status + debug: true \ No newline at end of file diff --git a/tools/osquery/example_osquery.conf b/tools/osquery/example_osquery.conf index cbf7bc9158..6acd590400 100644 --- a/tools/osquery/example_osquery.conf +++ b/tools/osquery/example_osquery.conf @@ -3,10 +3,13 @@ "tls_dump": "true", "disable_distributed": "false", "distributed_plugin": "tls", - "distributed_interval": 5, + "distributed_interval": 30, "distributed_tls_max_attempts": 3, - "distributed_tls_read_endpoint": "/api/v1/osquery/read", - "distributed_tls_write_endpoint": "/api/v1/osquery/write" + "distributed_tls_read_endpoint": "/api/v1/osquery/distributed/read", + "distributed_tls_write_endpoint": "/api/v1/osquery/distributed/write", + "logger_plugin": "tls", + "logger_tls_endpoint": "/api/v1/osquery/log", + "logger_tls_period": 5 }, "schedule": { @@ -15,7 +18,7 @@ // The exact query to run. "query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;", // The interval in seconds to run this query, not an exact interval. - "interval": 3600 + "interval": 5 } },