Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-23 17:08:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Ingest status and result logs (#77)

Browse source 
* Implement log endpoint for status/result log ingestion
* Define interface for log handlers: OsqueryResultHandler and OsqueryStatusHandler
* Initial implementation of file logger handlers
* Unit + integration tests

Closes #7
This commit is contained in:
Zachary Wasserman 2016-08-17 12:45:29 -07:00 committed by GitHub
parent 0c51890b30
commit 503ae54f46
13 changed files with 599 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -12,6 +12,7 @@ node_modules
app/bindata.go
*.cover
*.test
*.log
# operating system artifacts
.DS_Store

5
app/email_test.go
View file

@ -3,6 +3,7 @@ package app
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
@ -60,7 +61,7 @@ func (w *mockResponseWriter) WriteHeader(int) {
func TestUnauthenticatedPasswordReset(t *testing.T) {
db := openTestDB(t)
pool := newMockSMTPConnectionPool()
r := CreateServer(db, pool, &testLogger{t: t})
r := CreateServer(db, pool, &testLogger{t: t}, &OsqueryLogWriter{Writer: ioutil.Discard}, &OsqueryLogWriter{Writer: ioutil.Discard})
admin, _ := NewUser(db, "admin", "foobar", "admin@kolide.co", true, false)
{
@ -119,7 +120,7 @@ func TestUnauthenticatedPasswordReset(t *testing.T) {
func TestAuthenticatedPasswordReset(t *testing.T) {
db := openTestDB(t)
pool := newMockSMTPConnectionPool()
r := CreateServer(db, pool, &testLogger{t: t})
r := CreateServer(db, pool, &testLogger{t: t}, &OsqueryLogWriter{Writer: ioutil.Discard}, &OsqueryLogWriter{Writer: ioutil.Discard})
admin, _ := NewUser(db, "admin", "foobar", "admin@kolide.co", true, false)
request, _ := http.NewRequest("GET", "/", nil)
writer := newMockResponseWriter()

249
app/osquery.go
View file

@ -1,7 +1,9 @@
package app
import (
"encoding/json"
"fmt"
"io"
"net/http"
"time"
@ -12,6 +14,51 @@ import (
"github.com/spf13/viper"
)
// The "output plugin" interface for osquery result logs. The implementer of
// this interface can do whatever processing they would like with the log,
// returning the appropriate error status
type OsqueryResultHandler interface {
HandleResultLog(log OsqueryResultLog, nodeKey string) error
}
// The "output plugin" interface for osquery status logs. The implementer of
// this interface can do whatever processing they would like with the log,
// returning the appropriate error status
type OsqueryStatusHandler interface {
HandleStatusLog(log OsqueryStatusLog, nodeKey string) error
}
// This struct is used for injecting dependencies for osquery TLS processing.
// It can be configured in a `main` function to bind the appropriate handlers
// and it's methods can be attached to routes.
type OsqueryHandler struct {
ResultHandler OsqueryResultHandler
StatusHandler OsqueryStatusHandler
}
// Basic implementation of the `OsqueryResultHandler` and
// `OsqueryStatusHandler` interfaces. It will write the logs to the io.Writer
// provided in Writer.
type OsqueryLogWriter struct {
Writer io.Writer
}
func (w *OsqueryLogWriter) HandleStatusLog(log OsqueryStatusLog, nodeKey string) error {
err := json.NewEncoder(w.Writer).Encode(log)
if err != nil {
return errors.NewFromError(err, http.StatusInternalServerError, "error writing result log")
}
return nil
}
func (w *OsqueryLogWriter) HandleResultLog(log OsqueryResultLog, nodeKey string) error {
err := json.NewEncoder(w.Writer).Encode(log)
if err != nil {
return errors.NewFromError(err, http.StatusInternalServerError, "error writing status log")
}
return nil
}
type ScheduledQuery struct {
ID uint `gorm:"primary_key"`
CreatedAt time.Time
@ -159,26 +206,27 @@ type OsqueryConfigPostBody struct {
}
type OsqueryLogPostBody struct {
NodeKey string `json:"node_key" validate:"required"`
LogType string `json:"log_type" validate:"required"`
Data []map[string]interface{} `json:"data" validate:"required"`
NodeKey string `json:"node_key" validate:"required"`
LogType string `json:"log_type" validate:"required"`
Data *json.RawMessage `json:"data" validate:"required"`
}
type OsqueryResultLog struct {
Name string `json:"name"`
HostIdentifier string `json:"hostIdentifier"`
UnixTime string `json:"unixTime"`
CalendarTime string `json:"calendarTime"`
Name string `json:"name" validate:"required"`
HostIdentifier string `json:"hostIdentifier" validate:"required"`
UnixTime string `json:"unixTime" validate:"required"`
CalendarTime string `json:"calendarTime" validate:"required"`
Columns map[string]string `json:"columns"`
Action string `json:"action"`
Action string `json:"action" validate:"required"`
}
type OsqueryStatusLog struct {
Severity string `json:"severity"`
Filename string `json:"filename"`
Line string `json:"line"`
Message string `json:"message"`
Version string `json:"version"`
Severity string `json:"severity" validate:"required"`
Filename string `json:"filename" validate:"required"`
Line string `json:"line" validate:"required"`
Message string `json:"message" validate:"required"`
Version string `json:"version" validate:"required"`
Decorations map[string]string `json:"decorations"`
}
type OsqueryDistributedReadPostBody struct {
@ -244,12 +292,12 @@ func OsqueryEnroll(c *gin.Context) {
var body OsqueryEnrollPostBody
err := ParseAndValidateJSON(c, &body)
if err != nil {
errors.ReturnError(c, err)
errors.ReturnOsqueryError(c, err)
return
}
if body.EnrollSecret != viper.GetString("osquery.enroll_secret") {
errors.ReturnError(
errors.ReturnOsqueryError(
c,
errors.NewWithStatus(http.StatusUnauthorized,
"Node key invalid",
@ -263,7 +311,7 @@ func OsqueryEnroll(c *gin.Context) {
host, err := EnrollHost(db, body.HostIdentifier, "", "", "")
if err != nil {
errors.ReturnError(c, errors.DatabaseError(err))
errors.ReturnOsqueryError(c, errors.DatabaseError(err))
return
}
@ -280,7 +328,7 @@ func OsqueryConfig(c *gin.Context) {
var body OsqueryConfigPostBody
err := ParseAndValidateJSON(c, &body)
if err != nil {
errors.ReturnError(c, err)
errors.ReturnOsqueryError(c, err)
return
}
logrus.Debugf("OsqueryConfig: %s", body.NodeKey)
@ -297,77 +345,130 @@ func OsqueryConfig(c *gin.Context) {
})
}
func OsqueryLog(c *gin.Context) {
// Authenticate a (post-enrollment) TLS request from osqueryd. To do this we
// verify that the provided node key is valid.
func authenticateRequest(db *gorm.DB, nodeKey string) error {
host := Host{NodeKey: nodeKey}
err := db.Where(&host).First(&host).Error
if err != nil {
switch err {
case gorm.ErrRecordNotFound:
e := errors.NewFromError(
err,
http.StatusUnauthorized,
"Unauthorized",
)
// osqueryd expects the literal string "true" here
e.Extra = map[string]interface{}{"node_invalid": "true"}
return e
default:
return errors.DatabaseError(err)
}
}
return nil
}
// Unmarshal the status logs before sending them to the status log handler
func (h *OsqueryHandler) handleStatusLogs(db *gorm.DB, data *json.RawMessage, nodeKey string) error {
var statuses []OsqueryStatusLog
if err := json.Unmarshal(*data, &statuses); err != nil {
return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error")
}
// Perhaps we should validate the unmarshalled status log
for _, status := range statuses {
if err := h.StatusHandler.HandleStatusLog(status, nodeKey); err != nil {
return err
}
logrus.Debugf("Osquery status: %+v", status)
}
return nil
}
// Unmarshal the result logs before sending them to the result log handler
func (h *OsqueryHandler) handleResultLogs(db *gorm.DB, data *json.RawMessage, nodeKey string) error {
var results []OsqueryResultLog
if err := json.Unmarshal(*data, &results); err != nil {
return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error")
}
// Perhaps we should validate the unmarshalled result log
for _, result := range results {
if err := h.ResultHandler.HandleResultLog(result, nodeKey); err != nil {
return err
}
logrus.Debugf("Osquery result: %+v", result)
}
return nil
}
// Set the update time for the provided host to indicate that it has
// successfully checked in.
func updateLastSeen(db *gorm.DB, host *Host) error {
updateTime := time.Now()
err := db.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", updateTime, host.NodeKey).Error
if err != nil {
return errors.DatabaseError(err)
}
host.UpdatedAt = updateTime
return nil
}
// Endpoint used by the osqueryd TLS logger plugin
func (h *OsqueryHandler) OsqueryLog(c *gin.Context) {
var body OsqueryLogPostBody
err := ParseAndValidateJSON(c, &body)
if err != nil {
errors.ReturnError(c, err)
errors.ReturnOsqueryError(c, err)
return
}
logrus.Debugf("OsqueryLog: %s", body.LogType)
if body.LogType == "status" {
for _, data := range body.Data {
var log OsqueryStatusLog
db := GetDB(c)
severity, ok := data["severity"].(string)
if ok {
log.Severity = severity
} else {
logrus.Error("Error asserting the type of status log severity")
}
filename, ok := data["filename"].(string)
if ok {
log.Filename = filename
} else {
logrus.Error("Error asserting the type of status log filename")
}
line, ok := data["line"].(string)
if ok {
log.Line = line
} else {
logrus.Error("Error asserting the type of status log line")
}
message, ok := data["message"].(string)
if ok {
log.Message = message
} else {
logrus.Error("Error asserting the type of status log message")
}
version, ok := data["version"].(string)
if ok {
log.Version = version
} else {
logrus.Error("Error asserting the type of status log version")
}
logrus.WithFields(logrus.Fields{
"node_key": body.NodeKey,
"severity": log.Severity,
"filename": log.Filename,
"line": log.Line,
"version": log.Version,
}).Info(log.Message)
}
} else if body.LogType == "result" {
// TODO: handle all of the different kinds of results logs
err = authenticateRequest(db, body.NodeKey)
if err != nil {
errors.ReturnOsqueryError(c, err)
return
}
c.JSON(http.StatusOK,
gin.H{
"node_invalid": false,
})
switch body.LogType {
case "status":
err = h.handleStatusLogs(db, body.Data, body.NodeKey)
case "result":
err = h.handleResultLogs(db, body.Data, body.NodeKey)
default:
err = errors.NewWithStatus(
errors.StatusUnprocessableEntity,
"Unknown result type",
fmt.Sprintf("Unknown result type: %s", body.LogType),
)
}
if err != nil {
errors.ReturnOsqueryError(c, err)
return
}
err = updateLastSeen(db, &Host{NodeKey: body.NodeKey})
if err != nil {
errors.ReturnOsqueryError(c, err)
return
}
c.JSON(http.StatusOK, gin.H{})
}
func OsqueryDistributedRead(c *gin.Context) {
var body OsqueryDistributedReadPostBody
err := ParseAndValidateJSON(c, &body)
if err != nil {
errors.ReturnError(c, err)
errors.ReturnOsqueryError(c, err)
return
}
logrus.Debugf("OsqueryDistributedRead: %s", body.NodeKey)
@ -385,7 +486,7 @@ func OsqueryDistributedWrite(c *gin.Context) {
var body OsqueryDistributedWritePostBody
err := ParseAndValidateJSON(c, &body)
if err != nil {
errors.ReturnError(c, err)
errors.ReturnOsqueryError(c, err)
return
}
logrus.Debugf("OsqueryDistributedWrite: %s", body.NodeKey)

109
app/osquery_integration_test.go
View file

@ -9,7 +9,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestIntegrationEnrollHostBadSecret(t *testing.T) {
func TestOsqueryIntegrationEnrollHostBadSecret(t *testing.T) {
var req IntegrationRequests
req.New(t)
@ -33,7 +33,7 @@ func TestIntegrationEnrollHostBadSecret(t *testing.T) {
}
}
func TestIntegrationEnrollHostMissingIdentifier(t *testing.T) {
func TestOsqueryIntegrationEnrollHostMissingIdentifier(t *testing.T) {
var req IntegrationRequests
req.New(t)
@ -52,10 +52,10 @@ func TestIntegrationEnrollHostMissingIdentifier(t *testing.T) {
t.Fatalf("JSON decode error: %s JSON contents:\n %s", err.Error(), resp.Body.Bytes())
}
assert.Equal(t, "Validation error", body["message"])
assert.Equal(t, "Validation error", body["error"])
}
func TestIntegrationEnrollHostGood(t *testing.T) {
func TestOsqueryIntegrationEnrollHostGood(t *testing.T) {
var req IntegrationRequests
req.New(t)
@ -134,3 +134,104 @@ func TestIntegrationEnrollHostGood(t *testing.T) {
}
}
func TestOsqueryIntegrationOsqueryLogErrors(t *testing.T) {
var req IntegrationRequests
req.New(t)
// Missing log type should give error
data := json.RawMessage("{}")
resp := req.OsqueryLog("node_key", "", &data)
assert.Equal(t, errors.StatusUnprocessableEntity, resp.Code)
body := map[string]interface{}{}
assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body))
assert.Contains(t, body, "error")
// Bad node key should give error
resp = req.OsqueryLog("bad_node_key", "status", &data)
assert.Equal(t, http.StatusUnauthorized, resp.Code)
body = map[string]interface{}{}
assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body))
assert.Contains(t, body, "error")
assert.Contains(t, body, "node_invalid")
assert.Equal(t, "true", body["node_invalid"])
}
func TestOsqueryIntegrationOsqueryLogSuccess(t *testing.T) {
var req IntegrationRequests
req.New(t)
// First enroll
resp := req.EnrollHost("super secret", "fake_host_1")
body := map[string]interface{}{}
assert.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body))
assert.Equal(t, http.StatusOK, resp.Code)
assert.Contains(t, body, "node_key")
nodeKey := body["node_key"].(string)
// Now logging should be successful
expectStatus := []OsqueryStatusLog{
OsqueryStatusLog{
Severity: "bad",
Filename: "nope.cpp",
Line: "42",
Message: "bad stuff happened",
Version: "1.8.0",
Decorations: map[string]string{
"foo": "bar",
},
},
OsqueryStatusLog{
Severity: "worse",
Filename: "uhoh.cpp",
Line: "42",
Message: "bad stuff happened",
Version: "1.8.0",
Decorations: map[string]string{
"foo": "bar",
"baz": "bang",
},
},
}
expectResult := []OsqueryResultLog{
OsqueryResultLog{
Name: "query",
HostIdentifier: "somehost",
UnixTime: "the time",
CalendarTime: "other time",
Columns: map[string]string{
"foo": "bar",
"baz": "bang",
},
Action: "none",
},
}
// Send status log
jsonVal, err := json.Marshal(&expectStatus)
assert.NoError(t, err)
data := json.RawMessage(jsonVal)
resp = req.OsqueryLog(nodeKey, "status", &data)
assert.Equal(t, http.StatusOK, resp.Code)
// Send result log
jsonVal, err = json.Marshal(&expectResult)
assert.NoError(t, err)
data = json.RawMessage(jsonVal)
resp = req.OsqueryLog(nodeKey, "result", &data)
assert.Equal(t, http.StatusOK, resp.Code)
// Check that correct logs were logged
assert.Equal(t, expectStatus, req.statusHandler.Logs)
assert.Equal(t, expectResult, req.resultHandler.Logs)
}

179
app/osquery_test.go
View file

@ -1,7 +1,12 @@
package app
import (
"bytes"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestEnrollHost(t *testing.T) {
@ -91,3 +96,177 @@ func TestReEnrollHost(t *testing.T) {
}
}
func TestOsqueryLogWriterStatus(t *testing.T) {
buf := bytes.Buffer{}
logWriter := OsqueryLogWriter{Writer: &buf}
log := OsqueryStatusLog{
Severity: "bad",
Filename: "nope.cpp",
Line: "42",
Message: "bad stuff happened",
Version: "1.8.0",
Decorations: map[string]string{
"foo": "bar",
},
}
assert.NoError(t, logWriter.HandleStatusLog(log, "foo"))
jsonStr, err := json.Marshal(&log)
assert.NoError(t, err)
assert.Equal(t, string(jsonStr)+"\n", buf.String())
}
func TestOsqueryLogWriterResult(t *testing.T) {
buf := bytes.Buffer{}
logWriter := OsqueryLogWriter{Writer: &buf}
log := OsqueryResultLog{
Name: "query",
HostIdentifier: "somehost",
UnixTime: "the time",
CalendarTime: "other time",
Columns: map[string]string{
"foo": "bar",
"baz": "bang",
},
Action: "none",
}
assert.NoError(t, logWriter.HandleResultLog(log, "foo"))
jsonStr, err := json.Marshal(&log)
assert.NoError(t, err)
assert.Equal(t, string(jsonStr)+"\n", buf.String())
}
func TestOsqueryHandlerHandleStatusLogs(t *testing.T) {
writer := new(mockOsqueryStatusLogWriter)
handler := OsqueryHandler{StatusHandler: writer}
data := json.RawMessage("{")
assert.Error(t,
handler.handleStatusLogs(nil, &data, "foo"),
"should error with bad json",
)
expect := []OsqueryStatusLog{
OsqueryStatusLog{
Severity: "bad",
Filename: "nope.cpp",
Line: "42",
Message: "bad stuff happened",
Version: "1.8.0",
Decorations: map[string]string{
"foo": "bar",
},
},
OsqueryStatusLog{
Severity: "worse",
Filename: "uhoh.cpp",
Line: "42",
Message: "bad stuff happened",
Version: "1.8.0",
Decorations: map[string]string{
"foo": "bar",
"baz": "bang",
},
},
}
jsonVal, err := json.Marshal(&expect)
assert.NoError(t, err)
data = json.RawMessage(jsonVal)
assert.NoError(t, handler.handleStatusLogs(nil, &data, "foo"))
assert.Equal(t, expect, writer.Logs)
}
func TestOsqueryHandlerHandleResultLogs(t *testing.T) {
writer := new(mockOsqueryResultLogWriter)
handler := OsqueryHandler{ResultHandler: writer}
data := json.RawMessage("{")
assert.Error(t,
handler.handleResultLogs(nil, &data, "foo"),
"should error with bad json",
)
expect := []OsqueryResultLog{
OsqueryResultLog{
Name: "query",
HostIdentifier: "somehost",
UnixTime: "the time",
CalendarTime: "other time",
Columns: map[string]string{
"foo": "bar",
"baz": "bang",
},
Action: "none",
},
OsqueryResultLog{
Name: "other query",
HostIdentifier: "somehost",
UnixTime: "the time",
CalendarTime: "other time",
Columns: map[string]string{
"jim": "jam",
"foo": "bar",
"baz": "bang",
},
Action: "none",
},
}
jsonVal, err := json.Marshal(&expect)
assert.NoError(t, err)
data = json.RawMessage(jsonVal)
assert.NoError(t, handler.handleResultLogs(nil, &data, "foo"))
assert.Equal(t, expect, writer.Logs)
}
func TestOsqueryAuthenticateRequest(t *testing.T) {
db := openTestDB(t)
assert.Error(t, authenticateRequest(db, "foo"), "bad node key should fail")
host, err := EnrollHost(db, "fake_uuid", "fake_hostname", "", "")
assert.NoError(t, err, "enroll should succeed")
assert.NoError(t, authenticateRequest(db, host.NodeKey), "enrolled host should pass auth")
// re-enroll so the old node key is no longer valid
oldNodeKey := host.NodeKey
host, err = EnrollHost(db, "fake_uuid", "fake_hostname", "", "")
assert.NoError(t, err, "re-enroll should succeed")
// auth should fail now
assert.Error(t, authenticateRequest(db, oldNodeKey), "auth should succeed")
}
func TestOsqueryUpdateLastSeen(t *testing.T) {
db := openTestDB(t)
host, err := EnrollHost(db, "fake_uuid", "fake_hostname", "", "")
assert.NoError(t, err)
// Set update time to 0, then call update and make sure it is updated
assert.NoError(t, db.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", time.Time{}, host.NodeKey).Error)
// Clear and reload host
host = &Host{NodeKey: host.NodeKey}
assert.NoError(t, db.Where(&host).First(&host).Error)
assert.True(t, host.UpdatedAt.IsZero())
assert.NoError(t, updateLastSeen(db, host))
assert.False(t, host.UpdatedAt.IsZero())
// Make sure the update propagated to the DB
host = &Host{NodeKey: host.NodeKey}
assert.NoError(t, db.Where(&host).First(&host).Error)
assert.False(t, host.UpdatedAt.IsZero())
}

14
app/server.go
View file

@ -93,7 +93,7 @@ func NewSessionManager(c *gin.Context) *sessions.SessionManager {
func parseJSON(c *gin.Context, obj interface{}) error {
decoder := json.NewDecoder(c.Request.Body)
if err := decoder.Decode(obj); err != nil {
return err
return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error")
}
return nil
}
@ -102,7 +102,7 @@ func parseJSON(c *gin.Context, obj interface{}) error {
// the validator library.
func ParseAndValidateJSON(c *gin.Context, obj interface{}) error {
if err := parseJSON(c, obj); err != nil {
return errors.NewFromError(err, http.StatusBadRequest, "JSON parse error")
return err
}
return validate.Struct(obj)
@ -120,7 +120,7 @@ func NotFound(c *gin.Context) {
// CreateServer creates a gin.Engine HTTP server and configures it to be in a
// state such that it is ready to serve HTTP requests for the kolide application
func CreateServer(db *gorm.DB, pool SMTPConnectionPool, w io.Writer) *gin.Engine {
func CreateServer(db *gorm.DB, pool SMTPConnectionPool, w io.Writer, resultHandler OsqueryResultHandler, statusHandler OsqueryStatusHandler) *gin.Engine {
server := gin.New()
server.Use(DatabaseMiddleware(db))
server.Use(SMTPConnectionPoolMiddleware(pool))
@ -189,9 +189,15 @@ func CreateServer(db *gorm.DB, pool SMTPConnectionPool, w io.Writer) *gin.Engine
// osquery API endpoints
osq := v1.Group("/osquery")
osqueryHandler := OsqueryHandler{
ResultHandler: resultHandler,
StatusHandler: statusHandler,
}
osq.POST("/enroll", OsqueryEnroll)
osq.POST("/config", OsqueryConfig)
osq.POST("/log", OsqueryLog)
osq.POST("/log", osqueryHandler.OsqueryLog)
osq.POST("/distributed/read", OsqueryDistributedRead)
osq.POST("/distributed/write", OsqueryDistributedWrite)

57
app/test_util.go
View file

@ -43,11 +43,31 @@ func openTestDB(t *testing.T) *gorm.DB {
return db
}
type mockOsqueryStatusLogWriter struct {
Logs []OsqueryStatusLog
}
func (w *mockOsqueryStatusLogWriter) HandleStatusLog(log OsqueryStatusLog, nodeKey string) error {
w.Logs = append(w.Logs, log)
return nil
}
type mockOsqueryResultLogWriter struct {
Logs []OsqueryResultLog
}
func (w *mockOsqueryResultLogWriter) HandleResultLog(log OsqueryResultLog, nodeKey string) error {
w.Logs = append(w.Logs, log)
return nil
}
type IntegrationRequests struct {
r *gin.Engine
db *gorm.DB
pool SMTPConnectionPool
t *testing.T
r *gin.Engine
db *gorm.DB
pool SMTPConnectionPool
t *testing.T
statusHandler *mockOsqueryStatusLogWriter
resultHandler *mockOsqueryResultLogWriter
}
func (req *IntegrationRequests) New(t *testing.T) {
@ -56,6 +76,9 @@ func (req *IntegrationRequests) New(t *testing.T) {
req.db = openTestDB(t)
req.pool = newMockSMTPConnectionPool()
req.statusHandler = new(mockOsqueryStatusLogWriter)
req.resultHandler = new(mockOsqueryResultLogWriter)
// Until we have a better solution for first-user onboarding, manually
// create an admin
_, err := NewUser(req.db, "admin", "foobar", "admin@kolide.co", true, false)
@ -63,7 +86,8 @@ func (req *IntegrationRequests) New(t *testing.T) {
t.Fatalf("Error opening DB: %s", err.Error())
}
req.r = CreateServer(req.db, req.pool, &testLogger{t: t})
gin.SetMode(gin.ReleaseMode)
req.r = CreateServer(req.db, req.pool, &testLogger{t: t}, req.resultHandler, req.statusHandler)
}
func (req *IntegrationRequests) Login(username, password string, sessionOut *string) {
@ -476,3 +500,26 @@ func (req *IntegrationRequests) EnrollHost(enrollSecret, hostIdentifier string)
return response
}
func (req *IntegrationRequests) OsqueryLog(nodeKey, logType string, data *json.RawMessage) *httptest.ResponseRecorder {
response := httptest.NewRecorder()
body, err := json.Marshal(OsqueryLogPostBody{
NodeKey: nodeKey,
LogType: logType,
Data: data,
})
if err != nil {
req.t.Fatal(err.Error())
}
buff := new(bytes.Buffer)
buff.Write(body)
req.t.Log(buff.String())
request, _ := http.NewRequest("POST", "/api/v1/osquery/log", buff)
request.Header.Set("Content-Type", "application/json")
req.r.ServeHTTP(response, request)
return response
}

29
errors/errors.go
View file

@ -12,12 +12,15 @@ import (
// Kolide's internal representation for errors. It can be used to wrap another
// error (stored in Err), and additionally contains fields for public
// (PublicMessage) and private (PrivateMessage) error messages as well as the
// HTTP status code (StatusCode) corresponding to the error.
// HTTP status code (StatusCode) corresponding to the error. Extra holds extra
// information that will be inserted as top level key/value pairs in the error
// response.
type KolideError struct {
Err error
StatusCode int
PublicMessage string
PrivateMessage string
Extra map[string]interface{}
}
// Implementation of error interface
@ -73,11 +76,25 @@ const StatusUnprocessableEntity = 422
// Handle an error, printing debug information, writing to the HTTP response as
// appropriate for the dynamic error type.
func ReturnError(c *gin.Context, err error) {
baseReturnError(c, err, "message")
}
// osqueryd does not check the HTTP status code, and only looks for a
func ReturnOsqueryError(c *gin.Context, err error) {
baseReturnError(c, err, "error")
}
func baseReturnError(c *gin.Context, err error, messageKey string) {
switch typedErr := err.(type) {
case *KolideError:
c.JSON(typedErr.StatusCode,
gin.H{"message": typedErr.PublicMessage})
errJSON := map[string]interface{}{}
for key, val := range typedErr.Extra {
errJSON[key] = val
}
errJSON[messageKey] = typedErr.PublicMessage
c.JSON(typedErr.StatusCode, errJSON)
logrus.WithError(typedErr.Err).Debug(typedErr.PrivateMessage)
case validator.ValidationErrors:
@ -91,19 +108,19 @@ func ReturnError(c *gin.Context, err error) {
}
c.JSON(StatusUnprocessableEntity,
gin.H{"message": "Validation error",
gin.H{messageKey: "Validation error",
"errors": errors,
})
logrus.WithError(typedErr).Debug("Validation error")
case gorm.Errors, *gorm.Errors:
c.JSON(http.StatusInternalServerError,
gin.H{"message": "Database error"})
gin.H{messageKey: "Database error"})
logrus.WithError(typedErr).Debug(typedErr.Error())
default:
c.JSON(http.StatusInternalServerError,
gin.H{"message": "Unspecified error"})
gin.H{messageKey: "Unspecified error"})
logrus.WithError(typedErr).Debug("Unspecified error")
}
}

6
glide.lock generated
View file

@ -1,5 +1,5 @@
hash: cbd4502309ad1575151e0f9734937a85056213c326389ecc350259e677f2bfb6
updated: 2016-08-13T17:43:32.844377583-04:00
hash: 4ffa52b45a47295720fcf4dfcc0cc85c9f47a35a3bd43bdc6e0c2355bdd76109
updated: 2016-08-17T10:01:54.299187864-07:00
imports:
- name: github.com/alecthomas/template
version: a0175ee3bccc567396460bf5acd36800cb10c49c
@ -123,6 +123,8 @@ imports:
version: e5900212cbf65b181d3d8e08308ef06a01d117cf
- name: gopkg.in/go-playground/validator.v8
version: 5f57d2222ad794d0dffb07e664ea05e2ee07d60c
- name: gopkg.in/natefinch/lumberjack.v2
version: 514cbda263a734ae8caac038dadf05f8f3f9f738
- name: gopkg.in/yaml.v2
version: e4d366fc3c7938e2958e662b4258c7a89e1f0e3e
testImports: []

2
glide.yaml
View file

@ -74,3 +74,5 @@ import:
version: fd703108daeb23d77c124d12978e9b6c4f28f034
- package: github.com/spf13/cobra
- package: github.com/spf13/viper
- package: gopkg.in/natefinch/lumberjack.v2
version: v2.0

41
kolide.go
View file

@ -19,6 +19,7 @@ import (
"github.com/kolide/kolide-ose/app"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/natefinch/lumberjack.v2"
)
var (
@ -43,8 +44,8 @@ osquery management and orchestration
Configurable Options:
Options may be supplied in a yaml configuration file or via environment
variables. You only need to define the configuration values for which you
Options may be supplied in a yaml configuration file or via environment
variables. You only need to define the configuration values for which you
wish to override the default value.
Available Configurations:
@ -138,7 +139,33 @@ $7777777....$....$777$.....+DI..DDD..DDI...8D...D8......$D:..8D....8D...8D......
fmt.Println("Use Ctrl-C to stop")
fmt.Print("\n\n")
err = app.CreateServer(db, smtpConnectionPool, os.Stderr).RunTLS(
resultFile := viper.GetString("osquery.result_log_file")
resultHandler := &app.OsqueryLogWriter{
Writer: &lumberjack.Logger{
Filename: resultFile,
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, //days
},
}
statusFile := viper.GetString("osquery.status_log_file")
statusHandler := &app.OsqueryLogWriter{
Writer: &lumberjack.Logger{
Filename: statusFile,
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, //days
},
}
err = app.CreateServer(
db,
smtpConnectionPool,
os.Stderr,
resultHandler,
statusHandler,
).RunTLS(
viper.GetString("server.address"),
viper.GetString("server.cert"),
viper.GetString("server.key"),
@ -154,7 +181,7 @@ var prepareCmd = &cobra.Command{
Short: "Subcommands for initializing kolide infrastructure",
Long: `
Subcommands for initializing kolide infrastructure
To setup kolide infrastructure, use one of the available commands.
`,
Run: func(cmd *cobra.Command, args []string) {
@ -228,9 +255,9 @@ func initConfig() {
setDefaultConfigValue("mysql.password", "kolide")
setDefaultConfigValue("mysql.database", "kolide")
setDefaultConfigValue("server.address", "localhost:8080")
setDefaultConfigValue("server.address", "0.0.0.0:8080")
setDefaultConfigValue("app.web_address", "localhost:8080")
setDefaultConfigValue("app.web_address", "0.0.0.0:8080")
setDefaultConfigValue("auth.bcrypt_cost", 12)
setDefaultConfigValue("auth.salt_key_size", 24)
@ -243,6 +270,8 @@ func initConfig() {
setDefaultConfigValue("session.expiration_seconds", 60*60*24*90)
setDefaultConfigValue("osquery.node_key_size", 24)
setDefaultConfigValue("osquery.status_log_file", "/tmp/osquery_status")
setDefaultConfigValue("osquery.result_log_file", "/tmp/osquery_result")
if debug {
logrus.SetLevel(logrus.DebugLevel)

3
tools/app/kolide.yaml
View file

@ -5,4 +5,7 @@ auth:
jwt_key: very secure
osquery:
enroll_secret: super secure
result_log_file: /tmp/osquery_result
status_log_file: /tmp/osquery_status
debug: true

11
tools/osquery/example_osquery.conf
View file

@ -3,10 +3,13 @@
"tls_dump": "true",
"disable_distributed": "false",
"distributed_plugin": "tls",
"distributed_interval": 5,
"distributed_interval": 30,
"distributed_tls_max_attempts": 3,
"distributed_tls_read_endpoint": "/api/v1/osquery/read",
"distributed_tls_write_endpoint": "/api/v1/osquery/write"
"distributed_tls_read_endpoint": "/api/v1/osquery/distributed/read",
"distributed_tls_write_endpoint": "/api/v1/osquery/distributed/write",
"logger_plugin": "tls",
"logger_tls_endpoint": "/api/v1/osquery/log",
"logger_tls_period": 5
},
"schedule": {
@ -15,7 +18,7 @@
// The exact query to run.
"query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;",
// The interval in seconds to run this query, not an exact interval.
"interval": 3600
"interval": 5
}
},