From 1fa5ce16b82066f6d4bb630f72d25e780fdd28f2 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 15 Sep 2021 08:50:32 -0400 Subject: [PATCH 01/82] Add configurable Redis connection retries and following of cluster redirections (#2045) Closes #1969 --- changes/issue-1969-redis-config | 2 + cmd/fleet/serve.go | 20 +-- docs/2-Deploying/2-Configuration.md | 36 ++++- server/config/config.go | 34 ++-- server/datastore/redis/redis.go | 122 ++++++++++---- server/datastore/redis/redis_test.go | 176 ++++++++++++++++++++- server/fleet/redis_pool.go | 10 ++ server/live_query/redis_live_query.go | 4 +- server/live_query/redis_live_query_test.go | 14 +- server/pubsub/testing_utils.go | 14 +- server/sso/session_store.go | 6 +- server/sso/session_store_test.go | 14 +- 12 files changed, 386 insertions(+), 66 deletions(-) create mode 100644 changes/issue-1969-redis-config diff --git a/changes/issue-1969-redis-config b/changes/issue-1969-redis-config new file mode 100644 index 0000000000..dbbbcf214e --- /dev/null +++ b/changes/issue-1969-redis-config @@ -0,0 +1,2 @@ +* Add redis configuration option to retry failed connections. +* Add redis configuration option to follow cluster redirections. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 13dbf29b6c..d95beda2f3 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -200,21 +200,21 @@ the way that the Fleet server works. } } - redisPool, err := redis.NewRedisPool( - config.Redis.Address, - config.Redis.Password, - config.Redis.Database, - config.Redis.UseTLS, - config.Redis.ConnectTimeout, - config.Redis.KeepAlive, - ) + redisPool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: config.Redis.Address, + Password: config.Redis.Password, + Database: config.Redis.Database, + UseTLS: config.Redis.UseTLS, + ConnTimeout: config.Redis.ConnectTimeout, + KeepAlive: config.Redis.KeepAlive, + ConnectRetryAttempts: config.Redis.ConnectRetryAttempts, + ClusterFollowRedirections: config.Redis.ClusterFollowRedirections, + }) if err != nil { initFatal(err, "initialize Redis") } resultStore := pubsub.NewRedisQueryResults(redisPool, config.Redis.DuplicateResults) liveQueryStore := live_query.NewRedisLiveQuery(redisPool) - // TODO: should that only be done when a certain "migrate" flag is set, - // to prevent affecting every startup? if err := liveQueryStore.MigrateKeys(); err != nil { level.Info(logger).Log( "err", err, diff --git a/docs/2-Deploying/2-Configuration.md b/docs/2-Deploying/2-Configuration.md index 1a41211d35..b7da00f7f6 100644 --- a/docs/2-Deploying/2-Configuration.md +++ b/docs/2-Deploying/2-Configuration.md @@ -289,7 +289,7 @@ Maximum idle connections to database. This value should be equal to or less than max_idle_conns: 50 ``` -###### conn_max_lifetime +###### mysql_conn_max_lifetime Maximum amount of time, in seconds, a connection may be reused. @@ -358,7 +358,7 @@ Whether or not to duplicate Live Query results to another Redis channel named `L ###### redis_connect_timeout -Timeout for redis connection. +Timeout for redis connection. - Default value: 5s - Environment variable: `FLEET_REDIS_CONNECT_TIMEOUT` @@ -382,6 +382,38 @@ Interval between keep alive probes. keep_alive: 30s ``` +###### redis_connect_retry_attempts + +Maximum number of attempts to retry a failed connection to a redis node. Only +certain type of errors are retried, such as connection timeouts. + +- Default value: 0 (no retry) +- Environment variable: `FLEET_REDIS_CONNECT_RETRY_ATTEMPTS` +- Config file format: + + ``` + redis: + connect_retry_attempts: 2 + ``` + +###### redis_cluster_follow_redirections + +Whether or not to automatically follow redirection errors received from the +Redis server. Applies only to Redis Cluster setups, ignored in standalone +Redis. In Redis Cluster, keys can be moved around to different nodes when the +cluster is unstable and reorganizing the data. With this configuration option +set to true, those (typically short and transient) redirection errors can be +handled transparently instead of ending in an error. + +- Default value: false +- Environment variable: `FLEET_REDIS_CLUSTER_FOLLOW_REDIRECTIONS` +- Config file format: + + ``` + redis: + cluster_follow_redirections: true + ``` + ##### Server ###### server_address diff --git a/server/config/config.go b/server/config/config.go index e687581014..1d7f977c84 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -37,13 +37,15 @@ type MysqlConfig struct { // RedisConfig defines configs related to Redis type RedisConfig struct { - Address string - Password string - Database int - UseTLS bool `yaml:"use_tls"` - DuplicateResults bool `yaml:"duplicate_results"` - ConnectTimeout time.Duration `yaml:"connect_timeout"` - KeepAlive time.Duration `yaml:"keep_alive"` + Address string + Password string + Database int + UseTLS bool `yaml:"use_tls"` + DuplicateResults bool `yaml:"duplicate_results"` + ConnectTimeout time.Duration `yaml:"connect_timeout"` + KeepAlive time.Duration `yaml:"keep_alive"` + ConnectRetryAttempts int `yaml:"connect_retry_attempts"` + ClusterFollowRedirections bool `yaml:"cluster_follow_redirections"` } const ( @@ -243,6 +245,8 @@ func (man Manager) addConfigs() { man.addConfigBool("redis.duplicate_results", false, "Duplicate Live Query results to another Redis channel") man.addConfigDuration("redis.connect_timeout", 5*time.Second, "Timeout at connection time") man.addConfigDuration("redis.keep_alive", 10*time.Second, "Interval between keep alive probes") + man.addConfigInt("redis.connect_retry_attempts", 0, "Number of attempts to retry a failed connection") + man.addConfigBool("redis.cluster_follow_redirections", false, "Automatically follow Redis Cluster redirections") // Server man.addConfigString("server.address", "0.0.0.0:8080", @@ -417,13 +421,15 @@ func (man Manager) LoadConfig() FleetConfig { Mysql: loadMysqlConfig("mysql"), MysqlReadReplica: loadMysqlConfig("mysql_read_replica"), Redis: RedisConfig{ - Address: man.getConfigString("redis.address"), - Password: man.getConfigString("redis.password"), - Database: man.getConfigInt("redis.database"), - UseTLS: man.getConfigBool("redis.use_tls"), - DuplicateResults: man.getConfigBool("redis.duplicate_results"), - ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), - KeepAlive: man.getConfigDuration("redis.keep_alive"), + Address: man.getConfigString("redis.address"), + Password: man.getConfigString("redis.password"), + Database: man.getConfigInt("redis.database"), + UseTLS: man.getConfigBool("redis.use_tls"), + DuplicateResults: man.getConfigBool("redis.duplicate_results"), + ConnectTimeout: man.getConfigDuration("redis.connect_timeout"), + KeepAlive: man.getConfigDuration("redis.keep_alive"), + ConnectRetryAttempts: man.getConfigInt("redis.connect_retry_attempts"), + ClusterFollowRedirections: man.getConfigBool("redis.cluster_follow_redirections"), }, Server: ServerConfig{ Address: man.getConfigString("server.address"), diff --git a/server/datastore/redis/redis.go b/server/datastore/redis/redis.go index d9046af6e8..3679b7a61a 100644 --- a/server/datastore/redis/redis.go +++ b/server/datastore/redis/redis.go @@ -1,9 +1,11 @@ package redis import ( + "net" "strings" "time" + "github.com/cenkalti/backoff/v4" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" @@ -17,29 +19,64 @@ type standalonePool struct { addr string } +func (p *standalonePool) ConfigureDoer(conn redis.Conn) redis.Conn { + return conn +} + func (p *standalonePool) Stats() map[string]redis.PoolStats { return map[string]redis.PoolStats{ p.addr: p.Pool.Stats(), } } +type clusterPool struct { + *redisc.Cluster + followRedirs bool +} + +// ConfigureDoer configures conn to follow redirections if the redis +// configuration requested it. If the conn is already in error, or +// if it is not a redisc cluster connection, it is returned unaltered. +func (p *clusterPool) ConfigureDoer(conn redis.Conn) redis.Conn { + if err := conn.Err(); err == nil && p.followRedirs { + rc, err := redisc.RetryConn(conn, 3, 300*time.Millisecond) + if err == nil { + return rc + } + } + return conn +} + +// PoolConfig holds the redis pool configuration options. +type PoolConfig struct { + Server string + Password string + Database int + UseTLS bool + ConnTimeout time.Duration + KeepAlive time.Duration + ConnectRetryAttempts int + ClusterFollowRedirections bool + + // allows for testing dial retries and other dial-related scenarios + testRedisDialFunc func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) +} + // NewRedisPool creates a Redis connection pool using the provided server // address, password and database. -func NewRedisPool( - server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration, -) (fleet.RedisPool, error) { - cluster := newCluster(server, password, database, useTLS, connTimeout, keepAlive) +func NewRedisPool(config PoolConfig) (fleet.RedisPool, error) { + cluster := newCluster(config) if err := cluster.Refresh(); err != nil { if isClusterDisabled(err) || isClusterCommandUnknown(err) { // not a Redis Cluster setup, use a standalone Redis pool - pool, _ := cluster.CreatePool(server) + pool, _ := cluster.CreatePool(config.Server) cluster.Close() - return &standalonePool{pool, server}, nil + return &standalonePool{pool, config.Server}, nil } return nil, errors.Wrap(err, "refresh cluster") } - return cluster, nil + return &clusterPool{cluster, config.ClusterFollowRedirections}, nil } // SplitRedisKeysBySlot takes a list of redis keys and groups them by hash slot @@ -49,7 +86,7 @@ func NewRedisPool( // simply returns all keys in the same group (i.e. the top-level slice has a // length of 1). func SplitRedisKeysBySlot(pool fleet.RedisPool, keys ...string) [][]string { - if _, isCluster := pool.(*redisc.Cluster); isCluster { + if _, isCluster := pool.(*clusterPool); isCluster { return redisc.SplitBySlot(keys...) } return [][]string{keys} @@ -61,7 +98,7 @@ func SplitRedisKeysBySlot(pool fleet.RedisPool, keys ...string) [][]string { // of nodes stops and EachRedisNode returns that error. For standalone redis, // fn is called only once. func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { - if cluster, isCluster := pool.(*redisc.Cluster); isCluster { + if cluster, isCluster := pool.(*clusterPool); isCluster { return cluster.EachNode(false, func(_ string, conn redis.Conn) error { return fn(conn) }) @@ -72,35 +109,64 @@ func EachRedisNode(pool fleet.RedisPool, fn func(conn redis.Conn) error) error { return fn(conn) } -func newCluster(server, password string, database int, useTLS bool, connTimeout, keepAlive time.Duration) *redisc.Cluster { +func newCluster(config PoolConfig) *redisc.Cluster { + opts := []redis.DialOption{ + redis.DialDatabase(config.Database), + redis.DialUseTLS(config.UseTLS), + redis.DialConnectTimeout(config.ConnTimeout), + redis.DialKeepAlive(config.KeepAlive), + // Read/Write timeouts not set here because we may see results + // only rarely on the pub/sub channel. + } + if config.Password != "" { + opts = append(opts, redis.DialPassword(config.Password)) + } + + dialFn := redis.Dial + if config.testRedisDialFunc != nil { + dialFn = config.testRedisDialFunc + } + return &redisc.Cluster{ - StartupNodes: []string{server}, - CreatePool: func(server string, opts ...redis.DialOption) (*redis.Pool, error) { + StartupNodes: []string{config.Server}, + CreatePool: func(server string, _ ...redis.DialOption) (*redis.Pool, error) { return &redis.Pool{ MaxIdle: 3, IdleTimeout: 240 * time.Second, + Dial: func() (redis.Conn, error) { - c, err := redis.Dial( - "tcp", - server, - redis.DialDatabase(database), - redis.DialUseTLS(useTLS), - redis.DialConnectTimeout(connTimeout), - redis.DialKeepAlive(keepAlive), - // Read/Write timeouts not set here because we may see results - // only rarely on the pub/sub channel. - ) - if err != nil { - return nil, err + var conn redis.Conn + op := func() error { + c, err := dialFn("tcp", server, opts...) + + var netErr net.Error + if errors.As(err, &netErr) { + if netErr.Temporary() || netErr.Timeout() { + // retryable error + return err + } + } + if err != nil { + // at this point, this is a non-retryable error + return backoff.Permanent(err) + } + + // success, store the connection to use + conn = c + return nil } - if password != "" { - if _, err := c.Do("AUTH", password); err != nil { - c.Close() + + if config.ConnectRetryAttempts > 0 { + boff := backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(config.ConnectRetryAttempts)) + if err := backoff.Retry(op, boff); err != nil { return nil, err } + } else if err := op(); err != nil { + return nil, err } - return c, err + return conn, nil }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Minute { return nil diff --git a/server/datastore/redis/redis_test.go b/server/datastore/redis/redis_test.go index ad3da7d6b5..30b55a3c16 100644 --- a/server/datastore/redis/redis_test.go +++ b/server/datastore/redis/redis_test.go @@ -2,15 +2,171 @@ package redis import ( "fmt" + "io" + "runtime" "testing" "time" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" + "github.com/pkg/errors" "github.com/stretchr/testify/require" ) +type netError struct { + error + timeout bool + temporary bool + allowedCalls int // once this reaches 0, mockDial does not return an error + countCalls int +} + +func (t *netError) Timeout() bool { return t.timeout } +func (t *netError) Temporary() bool { return t.temporary } + +var errFromConn = errors.New("SUCCESS") + +type redisConn struct{} + +func (redisConn) Close() error { return errFromConn } +func (redisConn) Err() error { return errFromConn } +func (redisConn) Do(_ string, _ ...interface{}) (interface{}, error) { return nil, errFromConn } +func (redisConn) Send(_ string, _ ...interface{}) error { return errFromConn } +func (redisConn) Flush() error { return errFromConn } +func (redisConn) Receive() (interface{}, error) { return nil, errFromConn } + +func TestConnectRetry(t *testing.T) { + mockDial := func(err error) func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) { + return func(net, addr string, opts ...redis.DialOption) (redis.Conn, error) { + var ne *netError + if errors.As(err, &ne) { + ne.countCalls++ + if ne.allowedCalls <= 0 { + return redisConn{}, nil + } + ne.allowedCalls-- + } + return nil, err + } + } + + cases := []struct { + err error + retries int + wantCalls int + min, max time.Duration + }{ + // the min-max time intervals are based on the backoff default configuration as + // used in the Dial func of the redis pool. It starts with 500ms interval, + // multiplies by 1.5 on each attempt, and has a randomization of 0.5 that must + // be accounted for. Example ranges of intervals are given at + // https://github.com/fleetdm/fleet/pull/1962#issue-729635664 + // and were used to calculate the (approximate) expected range. + { + io.EOF, 0, 1, 0, 100 * time.Millisecond, + }, // non-retryable, no retry configured + { + &netError{error: io.EOF, timeout: true, allowedCalls: 10}, 0, 1, 0, 100 * time.Millisecond, + }, // retryable, but no retry configured + { + io.EOF, 3, 1, 0, 100 * time.Millisecond, + }, // non-retryable, retry configured + { + &netError{error: io.EOF, timeout: true, allowedCalls: 10}, 2, 3, 625 * time.Millisecond, 3500 * time.Millisecond, + }, // retryable, retry configured + { + &netError{error: io.EOF, temporary: true, allowedCalls: 10}, 2, 3, 625 * time.Millisecond, 3500 * time.Millisecond, + }, // retryable, retry configured + { + &netError{error: io.EOF, allowedCalls: 10}, 2, 1, 0, 100 * time.Millisecond, + }, // net error, but non-retryable + { + &netError{error: io.EOF, timeout: true, allowedCalls: 1}, 10, 2, 250 * time.Millisecond, 750 * time.Millisecond, + }, // retryable, but succeeded after one retry + } + for _, c := range cases { + t.Run(c.err.Error(), func(t *testing.T) { + start := time.Now() + _, err := NewRedisPool(PoolConfig{ + Server: "127.0.0.1:12345", + ConnectRetryAttempts: c.retries, + testRedisDialFunc: mockDial(c.err), + }) + diff := time.Since(start) + require.GreaterOrEqual(t, diff, c.min) + require.LessOrEqual(t, diff, c.max) + require.Error(t, err) + + wantErr := io.EOF + var ne *netError + if errors.As(c.err, &ne) { + require.Equal(t, c.wantCalls, ne.countCalls) + if ne.allowedCalls == 0 { + wantErr = errFromConn + } + } else { + require.Equal(t, c.wantCalls, 1) + } + + // the error is returned as part of the cluster.Refresh error, hence the + // check with Contains. + require.Contains(t, err.Error(), wantErr.Error()) + }) + } +} + +func TestRedisPoolConfigureDoer(t *testing.T) { + const prefix = "TestRedisPoolConfigureDoer:" + + t.Run("standalone", func(t *testing.T) { + pool, teardown := setupRedisForTest(t, false, false) + defer teardown() + + c1 := pool.Get() + defer c1.Close() + c2 := pool.ConfigureDoer(pool.Get()) + defer c2.Close() + + // both conns work equally well, get nil because keys do not exist, + // but no redirection error (this is standalone redis). + _, err := redis.String(c1.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c1.Do("GET", prefix+"{b}")) + require.Equal(t, redis.ErrNil, err) + + _, err = redis.String(c2.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c2.Do("GET", prefix+"{b}")) + require.Equal(t, redis.ErrNil, err) + }) + + t.Run("cluster", func(t *testing.T) { + pool, teardown := setupRedisForTest(t, true, true) + defer teardown() + + c1 := pool.Get() + defer c1.Close() + c2 := pool.ConfigureDoer(pool.Get()) + defer c2.Close() + + // unconfigured conn gets MOVED error on the second key + // (it is bound to {a}, {b} is on a different node) + _, err := redis.String(c1.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c1.Do("GET", prefix+"{b}")) + rerr := redisc.ParseRedir(err) + require.Error(t, rerr) + require.Equal(t, "MOVED", rerr.Type) + + // configured conn gets the nil value, it redirected automatically + _, err = redis.String(c2.Do("GET", prefix+"{a}")) + require.Equal(t, redis.ErrNil, err) + _, err = redis.String(c2.Do("GET", prefix+"{b}")) + require.Equal(t, redis.ErrNil, err) + }) +} + func TestEachRedisNode(t *testing.T) { const prefix = "TestEachRedisNode:" @@ -49,19 +205,23 @@ func TestEachRedisNode(t *testing.T) { } t.Run("standalone", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, false) + pool, teardown := setupRedisForTest(t, false, false) defer teardown() runTest(t, pool) }) t.Run("cluster", func(t *testing.T) { - pool, teardown := setupRedisForTest(t, true) + pool, teardown := setupRedisForTest(t, true, false) defer teardown() runTest(t, pool) }) } -func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardown func()) { +func setupRedisForTest(t *testing.T, cluster, redir bool) (pool fleet.RedisPool, teardown func()) { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + var ( addr = "127.0.0.1:" password = "" @@ -74,7 +234,15 @@ func setupRedisForTest(t *testing.T, cluster bool) (pool fleet.RedisPool, teardo } addr += port - pool, err := NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := NewRedisPool(PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + ClusterFollowRedirections: redir, + }) require.NoError(t, err) conn := pool.Get() diff --git a/server/fleet/redis_pool.go b/server/fleet/redis_pool.go index 18fa088b77..e1d86daf2c 100644 --- a/server/fleet/redis_pool.go +++ b/server/fleet/redis_pool.go @@ -5,7 +5,17 @@ import "github.com/gomodule/redigo/redis" // RedisPool is the common interface for redigo's Pool for standalone Redis // and redisc's Cluster for Redis Cluster. type RedisPool interface { + // Get returns a redis connection. It must always be closed after use. Get() redis.Conn + + // Close closes the redis connection. Close() error + + // Stats returns a map of redis pool statistics for each server address. Stats() map[string]redis.PoolStats + + // ConfigureDoer returns a redis connection that is properly configured + // to execute Do commands. This should only be called when the actions + // to execute are all done with conn.Do. + ConfigureDoer(redis.Conn) redis.Conn } diff --git a/server/live_query/redis_live_query.go b/server/live_query/redis_live_query.go index b81b88b3fb..a54e480b37 100644 --- a/server/live_query/redis_live_query.go +++ b/server/live_query/redis_live_query.go @@ -194,7 +194,7 @@ func (r *redisLiveQuery) RunQuery(name, sql string, hostIDs []uint) error { } func (r *redisLiveQuery) StopQuery(name string) error { - conn := r.pool.Get() + conn := r.pool.ConfigureDoer(r.pool.Get()) defer conn.Close() targetKey, sqlKey := generateKeys(name) @@ -279,7 +279,7 @@ func (r *redisLiveQuery) collectBatchQueriesForHost(hostID uint, queryKeys []str } func (r *redisLiveQuery) QueryCompletedByHost(name string, hostID uint) error { - conn := r.pool.Get() + conn := r.pool.ConfigureDoer(r.pool.Get()) defer conn.Close() targetKey, _ := generateKeys(name) diff --git a/server/live_query/redis_live_query_test.go b/server/live_query/redis_live_query_test.go index bdabe609f2..5a13411b32 100644 --- a/server/live_query/redis_live_query_test.go +++ b/server/live_query/redis_live_query_test.go @@ -1,6 +1,7 @@ package live_query import ( + "runtime" "testing" "time" @@ -99,6 +100,10 @@ func TestMigrateKeys(t *testing.T) { } func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, teardown func()) { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + var ( addr = "127.0.0.1:" password = "" @@ -111,7 +116,14 @@ func setupRedisLiveQuery(t *testing.T, cluster bool) (store *redisLiveQuery, tea } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + }) require.NoError(t, err) store = NewRedisLiveQuery(pool) diff --git a/server/pubsub/testing_utils.go b/server/pubsub/testing_utils.go index 2b8a02cc41..9ff5b39c1d 100644 --- a/server/pubsub/testing_utils.go +++ b/server/pubsub/testing_utils.go @@ -1,6 +1,7 @@ package pubsub import ( + "runtime" "testing" "time" @@ -10,6 +11,10 @@ import ( ) func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, teardown func()) { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + var ( addr = "127.0.0.1:" password = "" @@ -23,7 +28,14 @@ func SetupRedisForTest(t *testing.T, cluster bool) (store *redisQueryResults, te } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + }) require.NoError(t, err) store = NewRedisQueryResults(pool, dupResults) diff --git a/server/sso/session_store.go b/server/sso/session_store.go index b2277c5619..7b0bb36fdc 100644 --- a/server/sso/session_store.go +++ b/server/sso/session_store.go @@ -46,7 +46,7 @@ func (s *store) create(requestID, originalURL, metadata string, lifetimeSecs uin if len(requestID) < 8 { return errors.New("request id must be 8 or more characters in length") } - conn := s.pool.Get() + conn := s.pool.ConfigureDoer(s.pool.Get()) defer conn.Close() sess := Session{OriginalURL: originalURL, Metadata: metadata} var writer bytes.Buffer @@ -59,7 +59,7 @@ func (s *store) create(requestID, originalURL, metadata string, lifetimeSecs uin } func (s *store) Get(requestID string) (*Session, error) { - conn := s.pool.Get() + conn := s.pool.ConfigureDoer(s.pool.Get()) defer conn.Close() val, err := redis.String(conn.Do("GET", requestID)) if err != nil { @@ -81,7 +81,7 @@ func (s *store) Get(requestID string) (*Session, error) { var ErrSessionNotFound = errors.New("session not found") func (s *store) Expire(requestID string) error { - conn := s.pool.Get() + conn := s.pool.ConfigureDoer(s.pool.Get()) defer conn.Close() _, err := conn.Do("DEL", requestID) return err diff --git a/server/sso/session_store_test.go b/server/sso/session_store_test.go index 45b46af866..318ba12acd 100644 --- a/server/sso/session_store_test.go +++ b/server/sso/session_store_test.go @@ -2,6 +2,7 @@ package sso import ( "os" + "runtime" "testing" "time" @@ -12,6 +13,10 @@ import ( ) func newPool(t *testing.T, cluster bool) fleet.RedisPool { + if cluster && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") { + t.Skipf("docker networking limitations prevent running redis cluster tests on %s", runtime.GOOS) + } + if _, ok := os.LookupEnv("REDIS_TEST"); ok { var ( addr = "127.0.0.1:" @@ -25,7 +30,14 @@ func newPool(t *testing.T, cluster bool) fleet.RedisPool { } addr += port - pool, err := redis.NewRedisPool(addr, password, database, useTLS, 5*time.Second, 10*time.Second) + pool, err := redis.NewRedisPool(redis.PoolConfig{ + Server: addr, + Password: password, + Database: database, + UseTLS: useTLS, + ConnTimeout: 5 * time.Second, + KeepAlive: 10 * time.Second, + }) require.NoError(t, err) conn := pool.Get() defer conn.Close() From 0bc485b32fe253a3cb119ebf461e9f54ac597812 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Wed, 15 Sep 2021 11:21:26 -0400 Subject: [PATCH 02/82] fix some broken document separators in standard-query-library file (#2072) * fix some broken document separators in standard-query-library file * remove in progress dockerfile --- .../standard-query-library/standard-query-library.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml b/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml index 41d3e8eee0..a9f9017dce 100644 --- a/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml +++ b/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml @@ -446,13 +446,13 @@ spec: query: SELECT name, path, pid FROM processes WHERE on_disk = 0; purpose: Incident response contributors: alphabrevity - --- +--- apiVersion: v1 kind: query spec: name: Get user files matching a specific hash platforms: macOS, Linux - Description: Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.) + description: Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.) query: SELECT path,sha256 FROM hash WHERE path in (SELECT path FROM file WHERE size < 50000000 AND path LIKE ""/Users/%/Documents/%%"") AND sha256 = ""16d28cd1d78b823c4f961a6da78d67a8975d66cde68581798778ed1f98a56d75""; purpose: Informational contributors: alphabrevity @@ -466,13 +466,13 @@ spec: query: SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid = u.gid; purpose: Informational contributors: alphabrevity -—-- +--- apiVersion: v1 kind: query spec: name: Get all listening ports, by process platforms: Linux, macOS, Windows - Description: List ports that are listening on all interfaces, along with the process to which they are attached. + description: List ports that are listening on all interfaces, along with the process to which they are attached. query: SELECT lp.address, lp.pid, lp.port, lp.protocol, p.name, p.path, p.cmdline FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.address = "0.0.0.0"; purpose: Informational contributors: alphabrevity @@ -482,7 +482,7 @@ kind: query spec: name: Get whether TeamViewer is installed/running platforms: Windows - description: Description: Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine. + description: Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine. query: SELECT display_name,status,s.pid,p.path FROM services AS s JOIN processes AS p USING(pid) WHERE s.name LIKE "%teamviewer%"; purpose: Informational contributors: alphabrevity From e968a26b2d319b0656d94f7d16c59f43b1c1e9eb Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Wed, 15 Sep 2021 12:00:19 -0500 Subject: [PATCH 03/82] Make support challenges on Chrome OS explicit in docs (#2037) re https://github.com/fleetdm/fleet/issues/969 --- docs/1-Using-Fleet/12-Supported-browsers.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/1-Using-Fleet/12-Supported-browsers.md b/docs/1-Using-Fleet/12-Supported-browsers.md index 644545005c..5fae7fb5ea 100644 --- a/docs/1-Using-Fleet/12-Supported-browsers.md +++ b/docs/1-Using-Fleet/12-Supported-browsers.md @@ -19,4 +19,7 @@ We test each browser on Windows whenever possible, because our engineering team - Mobile Safari on iOS 10 - Mobile Chrome on Android 6 -**Note:** Mobile web is not yet supported in the Fleet Product. +### Note +> - Mobile web is not yet supported in the Fleet product. +> - The Fleet user interface [may not be fully supported](https://github.com/fleetdm/fleet/issues/969) in Google Chrome when the browser is running on Chrome OS + From e6368cc57fa20743c7210fd745799712fd33d253 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 15 Sep 2021 16:27:53 -0300 Subject: [PATCH 04/82] Refactor integration tests (#1821) * Refactor integration tests * Remove nopCloser and use io.NopCloser * Address review comments --- cmd/fleetctl/apply_test.go | 11 +- cmd/fleetctl/debug_test.go | 3 +- cmd/fleetctl/get_test.go | 15 +- cmd/fleetctl/hosts_test.go | 15 +- cmd/fleetctl/query_test.go | 3 +- cmd/fleetctl/users_test.go | 3 +- server/datastore/mysql/testing_utils.go | 9 + server/service/client.go | 2 +- server/service/http_auth_test.go | 13 +- server/service/integration_core_test.go | 392 ++++++++ server/service/integration_ds_only_test.go | 61 ++ server/service/integration_enterprise_test.go | 142 +++ server/service/integration_logger_test.go | 183 ++++ server/service/integration_test.go | 931 ------------------ server/service/testing_client.go | 143 +++ server/service/testing_utils.go | 25 + 16 files changed, 975 insertions(+), 976 deletions(-) create mode 100644 server/service/integration_core_test.go create mode 100644 server/service/integration_ds_only_test.go create mode 100644 server/service/integration_enterprise_test.go create mode 100644 server/service/integration_logger_test.go delete mode 100644 server/service/integration_test.go create mode 100644 server/service/testing_client.go diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 64cd4bfc21..84ec2fa996 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -41,8 +41,7 @@ var userRoleSpecList = []*fleet.User{ } func TestApplyUserRoles(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleSpecList, nil @@ -102,11 +101,10 @@ spec: func TestApplyTeamSpecs(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} - server, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) - defer server.Close() + _, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) teamsByName := map[string]*fleet.Team{ - "team1": &fleet.Team{ + "team1": { ID: 42, Name: "team1", Description: "team1 description", @@ -185,8 +183,7 @@ func writeTmpYml(t *testing.T, contents string) string { } func TestApplyAppConfig(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleSpecList, nil diff --git a/cmd/fleetctl/debug_test.go b/cmd/fleetctl/debug_test.go index f8a30a99b8..d71e72196e 100644 --- a/cmd/fleetctl/debug_test.go +++ b/cmd/fleetctl/debug_test.go @@ -44,8 +44,7 @@ oug6edBNpdhp8r2/4t6n3AouK0/zG2naAlmXV0JoFuEvy2bX0BbbbPg+v4WNZIsC func TestDebugConnectionCommand(t *testing.T) { t.Run("without certificate", func(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.VerifyEnrollSecretFunc = func(ctx context.Context, secret string) (*fleet.EnrollSecret, error) { return nil, errors.New("invalid") diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 8212b268fe..2e96b18ca6 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -55,8 +55,7 @@ var userRoleList = []*fleet.User{ } func TestGetUserRoles(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.ListUsersFunc = func(ctx context.Context, opt fleet.UserListOptions) ([]*fleet.User, error) { return userRoleList, nil @@ -114,8 +113,7 @@ func TestGetTeams(t *testing.T) { for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { license := tt.license - server, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) - defer server.Close() + _, ds := runServerWithMockedDS(t, service.TestServerOpts{License: license}) agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { @@ -196,8 +194,7 @@ spec: } func TestGetHosts(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) // this func is called when no host is specified i.e. `fleetctl get hosts --json` ds.ListHostsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.HostListOptions) ([]*fleet.Host, error) { @@ -343,8 +340,7 @@ func TestGetHosts(t *testing.T) { } func TestGetConfig(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{ @@ -412,8 +408,7 @@ spec: } func TestGetSoftawre(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) foo001 := fleet.Software{ Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "somecpe", diff --git a/cmd/fleetctl/hosts_test.go b/cmd/fleetctl/hosts_test.go index 7e8f080605..936f0cae63 100644 --- a/cmd/fleetctl/hosts_test.go +++ b/cmd/fleetctl/hosts_test.go @@ -10,8 +10,7 @@ import ( ) func TestHostTransferFlagChecks(t *testing.T) { - server, _ := runServerWithMockedDS(t) - defer server.Close() + runServerWithMockedDS(t) runAppCheckErr(t, []string{"hosts", "transfer", "--team", "team1", "--hosts", "host1", "--label", "AAA"}, @@ -24,8 +23,7 @@ func TestHostTransferFlagChecks(t *testing.T) { } func TestHostsTransferByHosts(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -48,8 +46,7 @@ func TestHostsTransferByHosts(t *testing.T) { } func TestHostsTransferByLabel(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -83,8 +80,7 @@ func TestHostsTransferByLabel(t *testing.T) { } func TestHostsTransferByStatus(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) @@ -118,8 +114,7 @@ func TestHostsTransferByStatus(t *testing.T) { } func TestHostsTransferByStatusAndSearchQuery(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.HostByIdentifierFunc = func(ctx context.Context, identifier string) (*fleet.Host, error) { require.Equal(t, "host1", identifier) diff --git a/cmd/fleetctl/query_test.go b/cmd/fleetctl/query_test.go index 306ecf1734..145d5d0f6e 100644 --- a/cmd/fleetctl/query_test.go +++ b/cmd/fleetctl/query_test.go @@ -16,8 +16,7 @@ import ( func TestLiveQuery(t *testing.T) { rs := pubsub.NewInmemQueryResults() lq := new(live_query.MockLiveQuery) - server, ds := runServerWithMockedDS(t, service.TestServerOpts{Rs: rs, Lq: lq}) - defer server.Close() + _, ds := runServerWithMockedDS(t, service.TestServerOpts{Rs: rs, Lq: lq}) ds.HostIDsByNameFunc = func(ctx context.Context, filter fleet.TeamFilter, hostnames []string) ([]uint, error) { return []uint{1234}, nil diff --git a/cmd/fleetctl/users_test.go b/cmd/fleetctl/users_test.go index 663c6514bc..482a1d4e0f 100644 --- a/cmd/fleetctl/users_test.go +++ b/cmd/fleetctl/users_test.go @@ -9,8 +9,7 @@ import ( ) func TestUserDelete(t *testing.T) { - server, ds := runServerWithMockedDS(t) - defer server.Close() + _, ds := runServerWithMockedDS(t) ds.UserByEmailFunc = func(ctx context.Context, email string) (*fleet.User, error) { return &fleet.User{ diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 6d84ca5e8a..be4d04a80e 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -243,3 +243,12 @@ func CreateMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datasto func CreateMySQLDS(t *testing.T) *Datastore { return createMySQLDSWithOptions(t, nil) } + +func CreateNamedMySQLDS(t *testing.T, name string) *Datastore { + if _, ok := os.LookupEnv("MYSQL_TEST"); !ok { + t.Skip("MySQL tests are disabled") + } + + t.Parallel() + return initializeDatabase(t, name, new(DatastoreTestOptions)) +} diff --git a/server/service/client.go b/server/service/client.go index abb98c15af..4e7a86b500 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -230,7 +230,7 @@ func (l *logRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { fmt.Fprintf(os.Stderr, "Read body error: %v", err) return nil, err } - res.Body = ioutil.NopCloser(resBody) + res.Body = io.NopCloser(resBody) return res, nil } diff --git a/server/service/http_auth_test.go b/server/service/http_auth_test.go index 3c9533de85..eea04c6164 100644 --- a/server/service/http_auth_test.go +++ b/server/service/http_auth_test.go @@ -21,7 +21,6 @@ import ( func TestLogin(t *testing.T) { ds, users, server := setupAuthTest(t) - defer server.Close() var loginTests = []struct { email string status int @@ -60,7 +59,7 @@ func TestLogin(t *testing.T) { j, err := json.Marshal(¶ms) assert.Nil(t, err) - requestBody := &nopCloser{bytes.NewBuffer(j)} + requestBody := io.NopCloser(bytes.NewBuffer(j)) resp, err := http.Post(server.URL+"/api/v1/fleet/login", "application/json", requestBody) require.Nil(t, err) assert.Equal(t, tt.status, resp.StatusCode) @@ -173,7 +172,7 @@ func getTestAdminToken(t *testing.T, server *httptest.Server) string { j, err := json.Marshal(¶ms) assert.Nil(t, err) - requestBody := &nopCloser{bytes.NewBuffer(j)} + requestBody := io.NopCloser(bytes.NewBuffer(j)) resp, err := http.Post(server.URL+"/api/v1/fleet/login", "application/json", requestBody) require.Nil(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) @@ -191,7 +190,6 @@ func getTestAdminToken(t *testing.T, server *httptest.Server) string { func TestNoHeaderErrorsDifferently(t *testing.T) { _, _, server := setupAuthTest(t) - defer server.Close() req, _ := http.NewRequest("GET", server.URL+"/api/v1/fleet/users", nil) client := &http.Client{} @@ -230,10 +228,3 @@ func TestNoHeaderErrorsDifferently(t *testing.T) { } `, string(bodyBytes)) } - -// an io.ReadCloser for new request body -type nopCloser struct { - io.Reader -} - -func (nopCloser) Close() error { return nil } diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go new file mode 100644 index 0000000000..3922225b70 --- /dev/null +++ b/server/service/integration_core_test.go @@ -0,0 +1,392 @@ +package service + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type integrationTestSuite struct { + suite.Suite + + withServer +} + +func (s *integrationTestSuite) SetupSuite() { + s.withServer.SetupSuite("integrationTestSuite") +} + +func TestIntegrations(t *testing.T) { + testingSuite := new(integrationTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +func (s *integrationTestSuite) TestDoubleUserCreationErrors() { + t := s.T() + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + GlobalRole: ptr.String(fleet.RoleObserver), + } + + s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusOK) + respSecond := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusConflict) + + assertBodyContains(t, respSecond, `Error 1062: Duplicate entry 'email@asd.com'`) +} + +func (s *integrationTestSuite) TestUserWithoutRoleErrors() { + t := s.T() + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + } + + resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) + assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "either global role or team role needs to be defined") +} + +func (s *integrationTestSuite) TestUserWithWrongRoleErrors() { + t := s.T() + + params := fleet.UserPayload{ + Name: ptr.String("user1"), + Email: ptr.String("email@asd.com"), + Password: ptr.String("pass"), + GlobalRole: ptr.String("wrongrole"), + } + resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) + assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "GlobalRole role can only be admin, observer, or maintainer.") +} + +func (s *integrationTestSuite) TestUserCreationWrongTeamErrors() { + t := s.T() + + teams := []fleet.UserTeam{ + { + Team: fleet.Team{ + ID: 9999, + }, + Role: fleet.RoleObserver, + }, + } + + params := fleet.UserPayload{ + Name: ptr.String("user2"), + Email: ptr.String("email2@asd.com"), + Password: ptr.String("pass"), + Teams: &teams, + } + resp := s.Do("POST", "/api/v1/fleet/users/admin", ¶ms, http.StatusUnprocessableEntity) + assertBodyContains(t, resp, `Error 1452: Cannot add or update a child row: a foreign key constraint fails`) +} + +func (s *integrationTestSuite) TestQueryCreationLogsActivity() { + t := s.T() + + admin1 := s.users["admin1@example.com"] + admin1.GravatarURL = "http://iii.com" + err := s.ds.SaveUser(context.Background(), &admin1) + require.NoError(t, err) + + params := fleet.QueryPayload{ + Name: ptr.String("user1"), + Query: ptr.String("select * from time;"), + } + s.Do("POST", "/api/v1/fleet/queries", ¶ms, http.StatusOK) + + activities := listActivitiesResponse{} + s.DoJSON("GET", "/api/v1/fleet/activities", nil, http.StatusOK, &activities) + + assert.Len(t, activities.Activities, 1) + assert.Equal(t, "Test Name admin1@example.com", activities.Activities[0].ActorFullName) + require.NotNil(t, activities.Activities[0].ActorGravatar) + assert.Equal(t, "http://iii.com", *activities.Activities[0].ActorGravatar) + assert.Equal(t, "created_saved_query", activities.Activities[0].Type) +} +func (s *integrationTestSuite) TestAppConfigAdditionalQueriesCanBeRemoved() { + t := s.T() + + spec := []byte(` + host_expiry_settings: + host_expiry_enabled: true + host_expiry_window: 0 + host_settings: + additional_queries: + time: SELECT * FROM time + enable_host_users: true +`) + s.applyConfig(spec) + + spec = []byte(` + host_settings: + enable_host_users: true + additional_queries: null +`) + s.applyConfig(spec) + + config := s.getConfig() + assert.Nil(t, config.HostSettings.AdditionalQueries) + assert.True(t, config.HostExpirySettings.HostExpiryEnabled) +} + +func (s *integrationTestSuite) TestAppConfigDefaultValues() { + config := s.getConfig() + s.Run("Update interval", func() { + require.Equal(s.T(), 1*time.Hour, config.UpdateInterval.OSQueryDetail) + }) + + s.Run("has logging", func() { + require.NotNil(s.T(), config.Logging) + }) +} + +func (s *integrationTestSuite) TestUserRolesSpec() { + t := s.T() + + _, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + + email := t.Name() + "@asd.com" + u := &fleet.User{ + Password: []byte("asd"), + Name: t.Name(), + Email: email, + GravatarURL: "http://asd.com", + GlobalRole: ptr.String(fleet.RoleObserver), + } + user, err := s.ds.NewUser(context.Background(), u) + require.NoError(t, err) + assert.Len(t, user.Teams, 0) + + spec := []byte(fmt.Sprintf(` + roles: + %s: + global_role: null + teams: + - role: maintainer + team: team1 +`, + email)) + + var userRoleSpec applyUserRoleSpecsRequest + err = yaml.Unmarshal(spec, &userRoleSpec.Spec) + require.NoError(t, err) + + s.Do("POST", "/api/v1/fleet/users/roles/spec", &userRoleSpec, http.StatusOK) + + user, err = s.ds.UserByEmail(context.Background(), email) + require.NoError(t, err) + require.Len(t, user.Teams, 1) + assert.Equal(t, fleet.RoleMaintainer, user.Teams[0].Role) +} + +func (s *integrationTestSuite) TestGlobalSchedule() { + t := s.T() + + gs := fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 0) + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{ + Name: "TestQuery1", + Description: "Some description", + Query: "select * from osquery;", + ObserverCanRun: true, + }) + require.NoError(t, err) + + gsParams := fleet.ScheduledQueryPayload{QueryID: ptr.Uint(qr.ID), Interval: ptr.Uint(42)} + r := globalScheduleQueryResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/schedule", gsParams, http.StatusOK, &r) + + gs = fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 1) + assert.Equal(t, uint(42), gs.GlobalSchedule[0].Interval) + assert.Equal(t, "TestQuery1", gs.GlobalSchedule[0].Name) + id := gs.GlobalSchedule[0].ID + + gs = fleet.GlobalSchedulePayload{} + gsParams = fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)} + s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), gsParams, http.StatusOK, &gs) + + gs = fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 1) + assert.Equal(t, uint(55), gs.GlobalSchedule[0].Interval) + + r = globalScheduleQueryResponse{} + s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), nil, http.StatusOK, &r) + + gs = fleet.GlobalSchedulePayload{} + s.DoJSON("GET", "/api/v1/fleet/global/schedule", nil, http.StatusOK, &gs) + require.Len(t, gs.GlobalSchedule, 0) +} + +func (s *integrationTestSuite) TestTranslator() { + t := s.T() + + payload := translatorResponse{} + params := translatorRequest{List: []fleet.TranslatePayload{ + { + Type: fleet.TranslatorTypeUserEmail, + Payload: fleet.StringIdentifierToIDPayload{Identifier: "admin1@example.com"}, + }, + }} + s.DoJSON("POST", "/api/v1/fleet/translate", ¶ms, http.StatusOK, &payload) + require.Len(t, payload.List, 1) + + assert.Equal(t, s.users[payload.List[0].Payload.Identifier].ID, payload.List[0].Payload.ID) +} + +func (s *integrationTestSuite) TestVulnerableSoftware() { + t := s.T() + + host, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + "1", + UUID: t.Name() + "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + require.NotNil(t, host) + + soft := fleet.HostSoftware{ + Modified: true, + Software: []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "apps"}, + }, + } + host.HostSoftware = soft + require.NoError(t, s.ds.SaveHostSoftware(context.Background(), host)) + require.NoError(t, s.ds.LoadHostSoftware(context.Background(), host)) + + soft1 := host.Software[0] + if soft1.Name != "bar" { + soft1 = host.Software[1] + } + + require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) + require.NoError(t, s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) + + resp := s.Do("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), nil, http.StatusOK) + bodyBytes, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + expectedJSONSoft2 := `"name": "bar", + "version": "0.0.3", + "source": "apps", + "generated_cpe": "somecpe", + "vulnerabilities": [ + { + "cve": "cve-123-123-132", + "details_link": "https://nvd.nist.gov/vuln/detail/cve-123-123-132" + } + ]` + expectedJSONSoft1 := `"name": "foo", + "version": "0.0.1", + "source": "chrome_extensions", + "generated_cpe": "", + "vulnerabilities": null` + // We are doing Contains instead of equals to test the output for software in particular + // ignoring other things like timestamps and things that are outside the cope of this ticket + assert.Contains(t, string(bodyBytes), expectedJSONSoft2) + assert.Contains(t, string(bodyBytes), expectedJSONSoft1) +} + +func (s *integrationTestSuite) TestGlobalPolicies() { + t := s.T() + + for i := 0; i < 3; i++ { + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), + OsqueryHostID: strconv.Itoa(i), + NodeKey: fmt.Sprintf("%d", i), + UUID: fmt.Sprintf("%d", i), + Hostname: fmt.Sprintf("foo.local%d", i), + }) + require.NoError(t, err) + } + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{ + Name: "TestQuery3", + Description: "Some description", + Query: "select * from osquery;", + ObserverCanRun: true, + }) + require.NoError(t, err) + + gpParams := globalPolicyRequest{QueryID: qr.ID} + gpResp := globalPolicyResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/policies", gpParams, http.StatusOK, &gpResp) + require.NotNil(t, gpResp.Policy) + assert.Equal(t, qr.ID, gpResp.Policy.QueryID) + + policiesResponse := listGlobalPoliciesResponse{} + s.DoJSON("GET", "/api/v1/fleet/global/policies", nil, http.StatusOK, &policiesResponse) + require.Len(t, policiesResponse.Policies, 1) + assert.Equal(t, qr.ID, policiesResponse.Policies[0].QueryID) + + singlePolicyResponse := getPolicyByIDResponse{} + singlePolicyURL := fmt.Sprintf("/api/v1/fleet/global/policies/%d", policiesResponse.Policies[0].ID) + s.DoJSON("GET", singlePolicyURL, nil, http.StatusOK, &singlePolicyResponse) + assert.Equal(t, qr.ID, singlePolicyResponse.Policy.QueryID) + assert.Equal(t, qr.Name, singlePolicyResponse.Policy.QueryName) + + listHostsURL := fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d", policiesResponse.Policies[0].ID) + listHostsResp := listHostsResponse{} + s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 3) + + h1 := listHostsResp.Hosts[0] + h2 := listHostsResp.Hosts[1] + + listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) + listHostsResp = listHostsResponse{} + s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 0) + + require.NoError(t, s.ds.RecordPolicyQueryExecutions(context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now())) + require.NoError(t, s.ds.RecordPolicyQueryExecutions(context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now())) + + listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) + listHostsResp = listHostsResponse{} + s.DoJSON("GET", listHostsURL, nil, http.StatusOK, &listHostsResp) + require.Len(t, listHostsResp.Hosts, 1) + + deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}} + deletePolicyResp := deleteGlobalPoliciesResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/policies/delete", deletePolicyParams, http.StatusOK, &deletePolicyResp) + + policiesResponse = listGlobalPoliciesResponse{} + s.DoJSON("GET", "/api/v1/fleet/global/policies", nil, http.StatusOK, &policiesResponse) + require.Len(t, policiesResponse.Policies, 0) +} diff --git a/server/service/integration_ds_only_test.go b/server/service/integration_ds_only_test.go new file mode 100644 index 0000000000..780a281fb2 --- /dev/null +++ b/server/service/integration_ds_only_test.go @@ -0,0 +1,61 @@ +package service + +import ( + "net/http" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type integrationDSTestSuite struct { + withDS + suite.Suite +} + +func TestIntegrationDSTestSuite(t *testing.T) { + testingSuite := new(integrationDSTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +func (s *integrationDSTestSuite) SetupSuite() { + s.withDS.SetupSuite("integrationDSTestSuite") +} + +func (s *integrationDSTestSuite) TestLicenseExpiration() { + testCases := []struct { + name string + tier string + expiration time.Time + shouldHaveHeader bool + }{ + {"basic expired", fleet.TierPremium, time.Now().Add(-24 * time.Hour), true}, + {"basic not expired", fleet.TierPremium, time.Now().Add(24 * time.Hour), false}, + {"core expired", fleet.TierFree, time.Now().Add(-24 * time.Hour), false}, + {"core not expired", fleet.TierFree, time.Now().Add(24 * time.Hour), false}, + } + + createTestUsers(s.T(), s.ds) + for _, tt := range testCases { + s.Run(tt.name, func() { + t := s.T() + + license := &fleet.LicenseInfo{Tier: tt.tier, Expiration: tt.expiration} + _, server := RunServerForTestsWithDS(t, s.ds, TestServerOpts{License: license, SkipCreateTestUsers: true}) + + ts := withServer{server: server} + ts.s = &s.Suite + ts.token = ts.getTestAdminToken() + + resp := ts.Do("GET", "/api/v1/fleet/config", nil, http.StatusOK) + if tt.shouldHaveHeader { + require.Equal(t, fleet.HeaderLicenseValueExpired, resp.Header.Get(fleet.HeaderLicenseKey)) + } else { + require.Equal(t, "", resp.Header.Get(fleet.HeaderLicenseKey)) + } + }) + } +} diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go new file mode 100644 index 0000000000..53af9f01b6 --- /dev/null +++ b/server/service/integration_enterprise_test.go @@ -0,0 +1,142 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func TestIntegrationsEnterprise(t *testing.T) { + testingSuite := new(integrationEnterpriseTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +type integrationEnterpriseTestSuite struct { + withServer + suite.Suite +} + +func (s *integrationEnterpriseTestSuite) SetupSuite() { + s.withDS.SetupSuite("integrationEnterpriseTestSuite") + + users, server := RunServerForTestsWithDS( + s.T(), s.ds, TestServerOpts{License: &fleet.LicenseInfo{Tier: fleet.TierPremium}}) + s.server = server + s.users = users + s.token = s.getTestAdminToken() +} + +func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { + t := s.T() + + // create a team through the service so it initializes the agent ops + teamName := t.Name() + "team1" + team := &fleet.Team{ + Name: teamName, + Description: "desc team1", + } + + s.Do("POST", "/api/v1/fleet/teams", team, http.StatusOK) + + // updates a team + agentOpts := json.RawMessage(`{"config": {"foo": "bar"}, "overrides": {"platforms": {"darwin": {"foo": "override"}}}}`) + teamSpecs := applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: teamName, AgentOptions: &agentOpts}}} + s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) + + team, err := s.ds.TeamByName(context.Background(), teamName) + require.NoError(t, err) + + assert.Len(t, team.Secrets, 0) + require.JSONEq(t, string(agentOpts), string(*team.AgentOptions)) + + // creates a team with default agent options + user, err := s.ds.UserByEmail(context.Background(), "admin1@example.com") + require.NoError(t, err) + + teams, err := s.ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) + require.NoError(t, err) + require.True(t, len(teams) >= 1) + + teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2"}}} + s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) + + teams, err = s.ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) + require.NoError(t, err) + assert.True(t, len(teams) >= 2) + + team, err = s.ds.TeamByName(context.Background(), "team2") + require.NoError(t, err) + + defaultOpts := `{"config": {"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}, "overrides": {}}` + assert.Len(t, team.Secrets, 0) + require.NotNil(t, team.AgentOptions) + require.JSONEq(t, defaultOpts, string(*team.AgentOptions)) + + // updates secrets + teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2", Secrets: []fleet.EnrollSecret{{Secret: "ABC"}}}}} + s.Do("POST", "/api/v1/fleet/spec/teams", teamSpecs, http.StatusOK) + + team, err = s.ds.TeamByName(context.Background(), "team2") + require.NoError(t, err) + + require.Len(t, team.Secrets, 1) + assert.Equal(t, "ABC", team.Secrets[0].Secret) +} + +func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { + t := s.T() + + team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + + ts := getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 0) + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery2", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) + require.NoError(t, err) + + gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} + r := teamScheduleQueryResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), gsParams, http.StatusOK, &r) + + ts = getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 1) + assert.Equal(t, uint(42), ts.Scheduled[0].Interval) + assert.Equal(t, "TestQuery2", ts.Scheduled[0].Name) + assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) + id := ts.Scheduled[0].ID + + modifyResp := modifyTeamScheduleResponse{} + modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} + s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), modifyParams, http.StatusOK, &modifyResp) + + // just to satisfy my paranoia, wanted to make sure the contents of the json would work + s.DoRaw("PATCH", fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), []byte(`{"interval": 77}`), http.StatusOK) + + ts = getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 1) + assert.Equal(t, uint(77), ts.Scheduled[0].Interval) + + deleteResp := deleteTeamScheduleResponse{} + s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), nil, http.StatusOK, &deleteResp) + + ts = getTeamScheduleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Scheduled, 0) +} diff --git a/server/service/integration_logger_test.go b/server/service/integration_logger_test.go new file mode 100644 index 0000000000..0816e6e8e8 --- /dev/null +++ b/server/service/integration_logger_test.go @@ -0,0 +1,183 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func TestIntegrationLoggerTestSuite(t *testing.T) { + testingSuite := new(integrationLoggerTestSuite) + testingSuite.s = &testingSuite.Suite + suite.Run(t, testingSuite) +} + +type integrationLoggerTestSuite struct { + withServer + suite.Suite + + buf *bytes.Buffer +} + +func (s *integrationLoggerTestSuite) SetupSuite() { + s.withDS.SetupSuite("integrationLoggerTestSuite") + + s.buf = new(bytes.Buffer) + logger := log.NewJSONLogger(s.buf) + logger = level.NewFilter(logger, level.AllowDebug()) + + users, server := RunServerForTestsWithDS(s.T(), s.ds, TestServerOpts{Logger: logger}) + s.server = server + s.users = users +} + +func (s *integrationLoggerTestSuite) TearDownTest() { + s.buf.Reset() +} + +func (s *integrationLoggerTestSuite) TestLogger() { + t := s.T() + + s.token = getTestAdminToken(t, s.server) + + s.getConfig() + + params := fleet.QueryPayload{ + Name: ptr.String("somequery"), + Description: ptr.String("desc"), + Query: ptr.String("select 1 from osquery;"), + } + payload := createQueryRequest{} + s.DoJSON("POST", "/api/v1/fleet/queries", params, http.StatusOK, &payload) + + logs := s.buf.String() + parts := strings.Split(strings.TrimSpace(logs), "\n") + assert.Len(t, parts, 3) + for i, part := range parts { + kv := make(map[string]string) + err := json.Unmarshal([]byte(part), &kv) + require.NoError(t, err) + + assert.NotEqual(t, "", kv["took"]) + + switch i { + case 0: + assert.Equal(t, "info", kv["level"]) + assert.Equal(t, "POST", kv["method"]) + assert.Equal(t, "/api/v1/fleet/login", kv["uri"]) + case 1: + assert.Equal(t, "debug", kv["level"]) + assert.Equal(t, "GET", kv["method"]) + assert.Equal(t, "/api/v1/fleet/config", kv["uri"]) + assert.Equal(t, "admin1@example.com", kv["user"]) + case 2: + assert.Equal(t, "info", kv["level"]) + assert.Equal(t, "POST", kv["method"]) + assert.Equal(t, "/api/v1/fleet/queries", kv["uri"]) + assert.Equal(t, "admin1@example.com", kv["user"]) + assert.Equal(t, "somequery", kv["name"]) + assert.Equal(t, "select 1 from osquery;", kv["sql"]) + default: + t.Fail() + } + } +} + +func (s *integrationLoggerTestSuite) TestOsqueryEndpointsLogErrors() { + t := s.T() + + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + "1234", + UUID: "1", + Hostname: "foo.local", + OsqueryHostID: t.Name(), + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + requestBody := io.NopCloser(bytes.NewBuffer([]byte(`{"node_key":"1234","log_type":"status","data":[}`))) + req, _ := http.NewRequest("POST", s.server.URL+"/api/v1/osquery/log", requestBody) + client := &http.Client{} + _, err = client.Do(req) + require.Nil(t, err) + + logString := s.buf.String() + assert.Equal(t, `{"err":"decoding JSON: invalid character '}' looking for beginning of value","level":"info","path":"/api/v1/osquery/log"} +`, logString) +} + +func (s *integrationLoggerTestSuite) TestSubmitStatusLog() { + t := s.T() + + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + OsqueryHostID: t.Name(), + }) + require.NoError(t, err) + + req := submitLogsRequest{ + NodeKey: "1234", + LogType: "status", + Data: nil, + } + res := submitLogsResponse{} + s.DoJSON("POST", "/api/v1/osquery/log", req, http.StatusOK, &res) + + logString := s.buf.String() + assert.Equal(t, 1, strings.Count(logString, "\"ip_addr\"")) + assert.Equal(t, 1, strings.Count(logString, "x_for_ip_addr")) +} + +func (s *integrationLoggerTestSuite) TestEnrollAgentLogsErrors() { + t := s.T() + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1234", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + }) + require.NoError(t, err) + + j, err := json.Marshal(&enrollAgentRequest{ + EnrollSecret: "1234", + HostIdentifier: "4321", + HostDetails: nil, + }) + require.NoError(t, err) + + s.DoRawNoAuth("POST", "/api/v1/osquery/enroll", j, http.StatusUnauthorized) + + parts := strings.Split(strings.TrimSpace(s.buf.String()), "\n") + require.Len(t, parts, 1) + logData := make(map[string]json.RawMessage) + require.NoError(t, json.Unmarshal([]byte(parts[0]), &logData)) + assert.Equal(t, json.RawMessage(`["enroll failed: no matching secret found"]`), logData["err"]) +} diff --git a/server/service/integration_test.go b/server/service/integration_test.go deleted file mode 100644 index ca24a626c2..0000000000 --- a/server/service/integration_test.go +++ /dev/null @@ -1,931 +0,0 @@ -package service - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server/datastore/mysql" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/fleetdm/fleet/v4/server/test" - "github.com/ghodss/yaml" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDoubleUserCreationErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - GlobalRole: ptr.String(fleet.RoleObserver), - } - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - requestBody = &nopCloser{bytes.NewBuffer(j)} - req, _ = http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - resp, err = client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusConflict, resp.StatusCode) - assertBodyContains(t, resp, `Error 1062: Duplicate entry 'email@asd.com'`) -} - -func TestUserWithoutRoleErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - } - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) - assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "either global role or team role needs to be defined") -} - -func TestUserWithWrongRoleErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - GlobalRole: ptr.String("wrongrole"), - } - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/fleet/users/admin", requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) - assertErrorCodeAndMessage(t, resp, fleet.ErrNoRoleNeeded, "GlobalRole role can only be admin, observer, or maintainer.") -} - -func TestUserCreationWrongTeamErrors(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - teams := []fleet.UserTeam{ - { - Team: fleet.Team{ - ID: 9999, - }, - Role: fleet.RoleObserver, - }, - } - - params := fleet.UserPayload{ - Name: ptr.String("user1"), - Email: ptr.String("email@asd.com"), - Password: ptr.String("pass"), - Teams: &teams, - } - method := "POST" - path := "/api/v1/fleet/users/admin" - expectedStatusCode := http.StatusUnprocessableEntity - - resp, closeFunc := doReq(t, params, method, server, path, token, expectedStatusCode) - defer closeFunc() - assertBodyContains(t, resp, `Error 1452: Cannot add or update a child row: a foreign key constraint fails`) -} - -func doReq( - t *testing.T, - params interface{}, - method string, - server *httptest.Server, - path string, - token string, - expectedStatusCode int, -) (*http.Response, func()) { - j, err := json.Marshal(¶ms) - assert.Nil(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest(method, server.URL+path, requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, expectedStatusCode, resp.StatusCode) - return resp, func() { - thisResp := resp - thisResp.Body.Close() - } -} - -func doRawReq( - t *testing.T, - body []byte, - method string, - server *httptest.Server, - path string, - token string, - expectedStatusCode int, -) *http.Response { - requestBody := &nopCloser{bytes.NewBuffer(body)} - req, _ := http.NewRequest(method, server.URL+path, requestBody) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - client := &http.Client{} - resp, err := client.Do(req) - require.Nil(t, err) - assert.Equal(t, expectedStatusCode, resp.StatusCode) - return resp -} - -func doJSONReq( - t *testing.T, - params interface{}, - method string, - server *httptest.Server, - path string, - token string, - expectedStatusCode int, - v interface{}, -) { - resp, closeFunc := doReq(t, params, method, server, path, token, expectedStatusCode) - defer closeFunc() - err := json.NewDecoder(resp.Body).Decode(v) - require.Nil(t, err) -} - -func assertBodyContains(t *testing.T, resp *http.Response, expectedError string) { - bodyBytes, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err) - bodyString := string(bodyBytes) - assert.Contains(t, bodyString, expectedError) -} - -func TestQueryCreationLogsActivity(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - users, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - admin1 := users["admin1@example.com"] - admin1.GravatarURL = "http://iii.com" - err := ds.SaveUser(context.Background(), &admin1) - require.NoError(t, err) - - params := fleet.QueryPayload{ - Name: ptr.String("user1"), - Query: ptr.String("select * from time;"), - } - _, closeFunc := doReq(t, params, "POST", server, "/api/v1/fleet/queries", token, http.StatusOK) - defer closeFunc() - type activitiesRespose struct { - Activities []map[string]interface{} `json:"activities"` - } - activities := activitiesRespose{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/activities", token, http.StatusOK, &activities) - - assert.Len(t, activities.Activities, 1) - assert.Equal(t, "Test Name admin1@example.com", activities.Activities[0]["actor_full_name"]) - assert.Equal(t, "http://iii.com", activities.Activities[0]["actor_gravatar"]) - assert.Equal(t, "created_saved_query", activities.Activities[0]["type"]) -} - -func getJSON(r *http.Response, target interface{}) error { - return json.NewDecoder(r.Body).Decode(target) -} - -func assertErrorCodeAndMessage(t *testing.T, resp *http.Response, code int, message string) { - err := &fleet.Error{} - require.Nil(t, getJSON(resp, err)) - assert.Equal(t, code, err.Code) - assert.Equal(t, message, err.Message) -} - -func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - spec := []byte(` - host_expiry_settings: - host_expiry_enabled: true - host_expiry_window: 0 - host_settings: - additional_queries: - time: SELECT * FROM time - enable_host_users: true -`) - applyConfig(t, spec, server, token) - - spec = []byte(` - host_settings: - enable_host_users: true - additional_queries: null -`) - applyConfig(t, spec, server, token) - - config := getConfig(t, server, token) - assert.Nil(t, config.HostSettings.AdditionalQueries) - assert.True(t, config.HostExpirySettings.HostExpiryEnabled) -} - -func TestAppConfigUpdateInterval(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - config := getConfig(t, server, token) - require.Equal(t, 1*time.Hour, config.UpdateInterval.OSQueryDetail) -} - -func TestAppConfigHasLogging(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - config := getConfig(t, server, token) - require.NotNil(t, config.Logging) -} - -func applyConfig(t *testing.T, spec []byte, server *httptest.Server, token string) { - var appConfigSpec interface{} - err := yaml.Unmarshal(spec, &appConfigSpec) - require.NoError(t, err) - - _, closeFunc := doReq(t, appConfigSpec, "PATCH", server, "/api/v1/fleet/config", token, http.StatusOK) - closeFunc() -} - -func getConfig(t *testing.T, server *httptest.Server, token string) *appConfigResponse { - var responseBody *appConfigResponse - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/config", token, http.StatusOK, &responseBody) - return responseBody -} - -func TestUserRolesSpec(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - _, err := ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1", - Description: "desc team1", - }) - require.NoError(t, err) - token := getTestAdminToken(t, server) - - user, err := ds.UserByEmail(context.Background(), "user1@example.com") - require.NoError(t, err) - assert.Len(t, user.Teams, 0) - - spec := []byte(` - roles: - user1@example.com: - global_role: null - teams: - - role: maintainer - team: team1 -`) - - var userRoleSpec applyUserRoleSpecsRequest - err = yaml.Unmarshal(spec, &userRoleSpec.Spec) - require.NoError(t, err) - - _, closeFunc := doReq(t, userRoleSpec, "POST", server, "/api/v1/fleet/users/roles/spec", token, http.StatusOK) - closeFunc() - - user, err = ds.UserByEmail(context.Background(), "user1@example.com") - require.NoError(t, err) - require.Len(t, user.Teams, 1) - assert.Equal(t, fleet.RoleMaintainer, user.Teams[0].Role) - - // But users are not deleted - users, err := ds.ListUsers(context.Background(), fleet.UserListOptions{}) - require.NoError(t, err) - assert.Len(t, users, 3) -} - -func TestGlobalSchedule(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - test.AddAllHostsLabel(t, ds) - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - gs := fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - assert.Len(t, gs.GlobalSchedule, 0) - - qr, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "TestQuery", - Description: "Some description", - Query: "select * from osquery;", - ObserverCanRun: true, - }) - require.NoError(t, err) - - gsParams := fleet.ScheduledQueryPayload{QueryID: ptr.Uint(qr.ID), Interval: ptr.Uint(42)} - r := globalScheduleQueryResponse{} - doJSONReq(t, gsParams, "POST", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &r) - require.Nil(t, r.Err) - - gs = fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 1) - assert.Equal(t, uint(42), gs.GlobalSchedule[0].Interval) - assert.Equal(t, "TestQuery", gs.GlobalSchedule[0].Name) - id := gs.GlobalSchedule[0].ID - - gs = fleet.GlobalSchedulePayload{} - gsParams = fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)} - doJSONReq( - t, gsParams, "PATCH", server, - fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), - token, http.StatusOK, &gs, - ) - - gs = fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 1) - assert.Equal(t, uint(55), gs.GlobalSchedule[0].Interval) - - r = globalScheduleQueryResponse{} - doJSONReq( - t, nil, "DELETE", server, - fmt.Sprintf("/api/v1/fleet/global/schedule/%d", id), - token, http.StatusOK, &r, - ) - require.Nil(t, r.Err) - - gs = fleet.GlobalSchedulePayload{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/schedule", token, http.StatusOK, &gs) - require.Len(t, gs.GlobalSchedule, 0) -} - -func TestTeamSpecs(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{License: &fleet.LicenseInfo{Tier: fleet.TierPremium}}) - defer server.Close() - token := getTestAdminToken(t, server) - - // create a team through the service so it initializes the agent ops - team := &fleet.Team{ - Name: "team1", - Description: "desc team1", - } - _, closeFunc := doReq(t, team, "POST", server, "/api/v1/fleet/teams", token, http.StatusOK) - defer closeFunc() - - // updates a team - agentOpts := json.RawMessage(`{"config": {"foo": "bar"}, "overrides": {"platforms": {"darwin": {"foo": "override"}}}}`) - teamSpecs := applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team1", AgentOptions: &agentOpts}}} - _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) - defer closeFunc() - - team, err := ds.TeamByName(context.Background(), "team1") - require.NoError(t, err) - - assert.Len(t, team.Secrets, 0) - require.JSONEq(t, string(agentOpts), string(*team.AgentOptions)) - - // creates a team with default agent options - user, err := ds.UserByEmail(context.Background(), "admin1@example.com") - require.NoError(t, err) - - teams, err := ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) - require.NoError(t, err) - assert.Len(t, teams, 1) - - teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2"}}} - _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) - defer closeFunc() - - teams, err = ds.ListTeams(context.Background(), fleet.TeamFilter{User: user}, fleet.ListOptions{}) - require.NoError(t, err) - assert.Len(t, teams, 2) - - team, err = ds.TeamByName(context.Background(), "team2") - require.NoError(t, err) - - defaultOpts := `{"config": {"options": {"logger_plugin": "tls", "pack_delimiter": "/", "logger_tls_period": 10, "distributed_plugin": "tls", "disable_distributed": false, "logger_tls_endpoint": "/api/v1/osquery/log", "distributed_interval": 10, "distributed_tls_max_attempts": 3}, "decorators": {"load": ["SELECT uuid AS host_uuid FROM system_info;", "SELECT hostname AS hostname FROM system_info;"]}}, "overrides": {}}` - assert.Len(t, team.Secrets, 0) - require.NotNil(t, team.AgentOptions) - require.JSONEq(t, defaultOpts, string(*team.AgentOptions)) - - // updates secrets - teamSpecs = applyTeamSpecsRequest{Specs: []*fleet.TeamSpec{{Name: "team2", Secrets: []fleet.EnrollSecret{{Secret: "ABC"}}}}} - _, closeFunc = doReq(t, teamSpecs, "POST", server, "/api/v1/fleet/spec/teams", token, http.StatusOK) - defer closeFunc() - - team, err = ds.TeamByName(context.Background(), "team2") - require.NoError(t, err) - - require.Len(t, team.Secrets, 1) - assert.Equal(t, "ABC", team.Secrets[0].Secret) -} - -func TestTranslator(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - users, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - payload := translatorResponse{} - params := translatorRequest{List: []fleet.TranslatePayload{ - { - Type: fleet.TranslatorTypeUserEmail, - Payload: fleet.StringIdentifierToIDPayload{Identifier: "admin1@example.com"}, - }, - }} - doJSONReq(t, ¶ms, "POST", server, "/api/v1/fleet/translate", token, http.StatusOK, &payload) - - require.Nil(t, payload.Err) - assert.Len(t, payload.List, 1) - - assert.Equal(t, users[payload.List[0].Payload.Identifier].ID, payload.List[0].Payload.ID) -} - -func TestTeamSchedule(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - test.AddAllHostsLabel(t, ds) - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - team1, err := ds.NewTeam(context.Background(), &fleet.Team{ - ID: 42, - Name: "team1", - Description: "desc team1", - }) - require.NoError(t, err) - - ts := getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - assert.Len(t, ts.Scheduled, 0) - - qr, err := ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) - require.NoError(t, err) - - gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} - r := teamScheduleQueryResponse{} - doJSONReq(t, gsParams, "POST", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &r) - require.Nil(t, r.Err) - - ts = getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - require.Len(t, ts.Scheduled, 1) - assert.Equal(t, uint(42), ts.Scheduled[0].Interval) - assert.Equal(t, "TestQuery", ts.Scheduled[0].Name) - assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) - id := ts.Scheduled[0].ID - - modifyResp := modifyTeamScheduleResponse{} - modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} - doJSONReq( - t, modifyParams, "PATCH", server, - fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), - token, http.StatusOK, &modifyResp, - ) - - // just to satisfy my paranoia, wanted to make sure the contents of the json would work - doRawReq(t, []byte(`{"interval": 77}`), "PATCH", server, - fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), - token, http.StatusOK) - - ts = getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - assert.Len(t, ts.Scheduled, 1) - assert.Equal(t, uint(77), ts.Scheduled[0].Interval) - - deleteResp := deleteTeamScheduleResponse{} - doJSONReq( - t, nil, "DELETE", server, - fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), - token, http.StatusOK, &deleteResp, - ) - require.Nil(t, r.Err) - - ts = getTeamScheduleResponse{} - doJSONReq(t, nil, "GET", server, fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), token, http.StatusOK, &ts) - assert.Len(t, ts.Scheduled, 0) -} - -func TestLogger(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - token := getTestAdminToken(t, server) - - getConfig(t, server, token) - params := fleet.QueryPayload{ - Name: ptr.String("somequery"), - Description: ptr.String("desc"), - Query: ptr.String("select 1 from osquery;"), - } - payload := createQueryRequest{} - doJSONReq(t, params, "POST", server, "/api/v1/fleet/queries", token, http.StatusOK, &payload) - - logs := buf.String() - parts := strings.Split(strings.TrimSpace(logs), "\n") - assert.Len(t, parts, 3) - for i, part := range parts { - kv := make(map[string]string) - err := json.Unmarshal([]byte(part), &kv) - require.NoError(t, err) - - assert.NotEqual(t, "", kv["took"]) - - switch i { - case 0: - assert.Equal(t, "info", kv["level"]) - assert.Equal(t, "POST", kv["method"]) - assert.Equal(t, "/api/v1/fleet/login", kv["uri"]) - case 1: - assert.Equal(t, "debug", kv["level"]) - assert.Equal(t, "GET", kv["method"]) - assert.Equal(t, "/api/v1/fleet/config", kv["uri"]) - assert.Equal(t, "admin1@example.com", kv["user"]) - case 2: - assert.Equal(t, "info", kv["level"]) - assert.Equal(t, "POST", kv["method"]) - assert.Equal(t, "/api/v1/fleet/queries", kv["uri"]) - assert.Equal(t, "admin1@example.com", kv["user"]) - assert.Equal(t, "somequery", kv["name"]) - assert.Equal(t, "select 1 from osquery;", kv["sql"]) - default: - t.Fail() - } - } -} - -func TestVulnerableSoftware(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - host, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - require.NotNil(t, host) - - soft := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, - {Name: "bar", Version: "0.0.3", Source: "apps"}, - }, - } - host.HostSoftware = soft - require.NoError(t, ds.SaveHostSoftware(context.Background(), host)) - require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) - - soft1 := host.Software[0] - if soft1.Name != "bar" { - soft1 = host.Software[1] - } - - require.NoError(t, ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) - - path := fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID) - resp, closeFunc := doReq(t, nil, "GET", server, path, token, http.StatusOK) - defer closeFunc() - bodyBytes, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - expectedJSONSoft2 := `"name": "bar", - "version": "0.0.3", - "source": "apps", - "generated_cpe": "somecpe", - "vulnerabilities": [ - { - "cve": "cve-123-123-132", - "details_link": "https://nvd.nist.gov/vuln/detail/cve-123-123-132" - } - ]` - expectedJSONSoft1 := `"name": "foo", - "version": "0.0.1", - "source": "chrome_extensions", - "generated_cpe": "", - "vulnerabilities": null` - // We are doing Contains instead of equals to test the output for software in particular - // ignoring other things like timestamps and things that are outside the cope of this ticket - assert.Contains(t, string(bodyBytes), expectedJSONSoft2) - assert.Contains(t, string(bodyBytes), expectedJSONSoft1) -} - -func TestGlobalPolicies(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds) - defer server.Close() - token := getTestAdminToken(t, server) - - for i := 0; i < 3; i++ { - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), - OsqueryHostID: strconv.Itoa(i), - NodeKey: fmt.Sprintf("%d", i), - UUID: fmt.Sprintf("%d", i), - Hostname: fmt.Sprintf("foo.local%d", i), - }) - require.NoError(t, err) - } - - qr, err := ds.NewQuery(context.Background(), &fleet.Query{ - Name: "TestQuery", - Description: "Some description", - Query: "select * from osquery;", - ObserverCanRun: true, - }) - require.NoError(t, err) - - gpParams := globalPolicyRequest{QueryID: qr.ID} - gpResp := globalPolicyResponse{} - doJSONReq(t, gpParams, "POST", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &gpResp) - require.NotNil(t, gpResp.Policy) - assert.Equal(t, qr.ID, gpResp.Policy.QueryID) - - policiesResponse := listGlobalPoliciesResponse{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &policiesResponse) - require.Len(t, policiesResponse.Policies, 1) - assert.Equal(t, qr.ID, policiesResponse.Policies[0].QueryID) - - singlePolicyResponse := getPolicyByIDResponse{} - singlePolicyURL := fmt.Sprintf("/api/v1/fleet/global/policies/%d", policiesResponse.Policies[0].ID) - doJSONReq(t, nil, "GET", server, singlePolicyURL, token, http.StatusOK, &singlePolicyResponse) - assert.Equal(t, qr.ID, singlePolicyResponse.Policy.QueryID) - assert.Equal(t, qr.Name, singlePolicyResponse.Policy.QueryName) - - listHostsURL := fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d", policiesResponse.Policies[0].ID) - listHostsResp := listHostsResponse{} - doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 3) - - h1 := listHostsResp.Hosts[0] - h2 := listHostsResp.Hosts[1] - - listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) - listHostsResp = listHostsResponse{} - doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 0) - - require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), h1.Host, map[uint]*bool{policiesResponse.Policies[0].ID: ptr.Bool(true)}, time.Now())) - require.NoError(t, ds.RecordPolicyQueryExecutions(context.Background(), h2.Host, map[uint]*bool{policiesResponse.Policies[0].ID: nil}, time.Now())) - - listHostsURL = fmt.Sprintf("/api/v1/fleet/hosts?policy_id=%d&policy_response=passing", policiesResponse.Policies[0].ID) - listHostsResp = listHostsResponse{} - doJSONReq(t, nil, "GET", server, listHostsURL, token, http.StatusOK, &listHostsResp) - require.Len(t, listHostsResp.Hosts, 1) - - deletePolicyParams := deleteGlobalPoliciesRequest{IDs: []uint{policiesResponse.Policies[0].ID}} - deletePolicyResp := deleteGlobalPoliciesResponse{} - doJSONReq(t, deletePolicyParams, "POST", server, "/api/v1/fleet/global/policies/delete", token, http.StatusOK, &deletePolicyResp) - - policiesResponse = listGlobalPoliciesResponse{} - doJSONReq(t, nil, "GET", server, "/api/v1/fleet/global/policies", token, http.StatusOK, &policiesResponse) - require.Len(t, policiesResponse.Policies, 0) -} - -func TestOsqueryEndpointsLogErrors(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - requestBody := &nopCloser{bytes.NewBuffer([]byte(`{"node_key":"1234","log_type":"status","data":[}`))} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/osquery/log", requestBody) - client := &http.Client{} - _, err = client.Do(req) - require.Nil(t, err) - - logString := buf.String() - assert.Equal(t, `{"err":"decoding JSON: invalid character '}' looking for beginning of value","level":"info","path":"/api/v1/osquery/log"} -`, logString) -} - -func TestSubmitStatusLog(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - token := getTestAdminToken(t, server) - - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - req := submitLogsRequest{ - NodeKey: "1234", - LogType: "status", - Data: nil, - } - res := submitLogsResponse{} - doJSONReq(t, req, "POST", server, "/api/v1/osquery/log", token, http.StatusOK, &res) - - logString := buf.String() - assert.Equal(t, 1, strings.Count(logString, "\"ip_addr\"")) - assert.Equal(t, 1, strings.Count(logString, "x_for_ip_addr")) -} - -func TestEnrollAgentLogsErrors(t *testing.T) { - buf := new(bytes.Buffer) - logger := log.NewJSONLogger(buf) - logger = level.NewFilter(logger, level.AllowDebug()) - - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{Logger: logger}) - defer server.Close() - - _, err := ds.NewHost(context.Background(), &fleet.Host{ - DetailUpdatedAt: time.Now(), - LabelUpdatedAt: time.Now(), - SeenTime: time.Now(), - NodeKey: "1234", - UUID: "1", - Hostname: "foo.local", - PrimaryIP: "192.168.1.1", - PrimaryMac: "30-65-EC-6F-C4-58", - }) - require.NoError(t, err) - - j, err := json.Marshal(&enrollAgentRequest{ - EnrollSecret: "1234", - HostIdentifier: "4321", - HostDetails: nil, - }) - require.NoError(t, err) - - requestBody := &nopCloser{bytes.NewBuffer(j)} - req, _ := http.NewRequest("POST", server.URL+"/api/v1/osquery/enroll", requestBody) - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(t, err) - require.NoError(t, resp.Body.Close()) - - parts := strings.Split(strings.TrimSpace(buf.String()), "\n") - require.Len(t, parts, 1) - logData := make(map[string]json.RawMessage) - require.NoError(t, json.Unmarshal([]byte(parts[0]), &logData)) - assert.Equal(t, json.RawMessage(`["enroll failed: no matching secret found"]`), logData["err"]) -} - -func TestLicenseExpiration(t *testing.T) { - ds := mysql.CreateMySQLDS(t) - defer ds.Close() - - testCases := []struct { - name string - tier string - expiration time.Time - shouldHaveHeader bool - }{ - {"premium expired", fleet.TierPremium, time.Now().Add(-24 * time.Hour), true}, - {"premium not expired", fleet.TierPremium, time.Now().Add(24 * time.Hour), false}, - {"free expired", fleet.TierFree, time.Now().Add(-24 * time.Hour), false}, - {"free not expired", fleet.TierFree, time.Now().Add(24 * time.Hour), false}, - } - - _ = createTestUsers(t, ds) - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - license := &fleet.LicenseInfo{Tier: tt.tier, Expiration: tt.expiration} - _, server := RunServerForTestsWithDS(t, ds, TestServerOpts{License: license, SkipCreateTestUsers: true}) - defer server.Close() - - token := getTestAdminToken(t, server) - - resp, closeFunc := doReq(t, nil, "GET", server, "/api/v1/fleet/config", token, http.StatusOK) - defer closeFunc() - if tt.shouldHaveHeader { - require.Equal(t, fleet.HeaderLicenseValueExpired, resp.Header.Get(fleet.HeaderLicenseKey)) - } else { - require.Equal(t, "", resp.Header.Get(fleet.HeaderLicenseKey)) - } - }) - } -} diff --git a/server/service/testing_client.go b/server/service/testing_client.go new file mode 100644 index 0000000000..e5c1e3be7b --- /dev/null +++ b/server/service/testing_client.go @@ -0,0 +1,143 @@ +package service + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + + "github.com/fleetdm/fleet/v4/server/datastore/mysql" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type withDS struct { + s *suite.Suite + ds *mysql.Datastore +} + +func (ts *withDS) SetupSuite(dbName string) { + ts.ds = mysql.CreateNamedMySQLDS(ts.s.T(), dbName) + test.AddAllHostsLabel(ts.s.T(), ts.ds) +} + +func (ts *withDS) TearDownSuite() { + ts.ds.Close() +} + +type withServer struct { + withDS + + server *httptest.Server + users map[string]fleet.User + token string +} + +func (ts *withServer) SetupSuite(dbName string) { + ts.withDS.SetupSuite(dbName) + + users, server := RunServerForTestsWithDS(ts.s.T(), ts.ds) + ts.server = server + ts.users = users + ts.token = ts.getTestAdminToken() +} + +func (ts *withServer) TearDownSuite() { + ts.withDS.TearDownSuite() +} + +func (ts *withServer) Do(verb, path string, params interface{}, expectedStatusCode int) *http.Response { + t := ts.s.T() + + j, err := json.Marshal(params) + require.NoError(t, err) + + resp := ts.DoRaw(verb, path, j, expectedStatusCode) + + t.Cleanup(func() { + resp.Body.Close() + }) + return resp +} + +func (ts *withServer) DoRawWithHeaders(verb string, path string, rawBytes []byte, expectedStatusCode int, headers map[string]string) *http.Response { + t := ts.s.T() + + requestBody := io.NopCloser(bytes.NewBuffer(rawBytes)) + req, _ := http.NewRequest(verb, ts.server.URL+path, requestBody) + for key, val := range headers { + req.Header.Add(key, val) + } + client := &http.Client{} + resp, err := client.Do(req) + require.NoError(t, err) + require.Equal(t, expectedStatusCode, resp.StatusCode) + + return resp +} + +func (ts *withServer) DoRaw(verb string, path string, rawBytes []byte, expectedStatusCode int) *http.Response { + return ts.DoRawWithHeaders(verb, path, rawBytes, expectedStatusCode, map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", ts.token), + }) +} + +func (ts *withServer) DoRawNoAuth(verb string, path string, rawBytes []byte, expectedStatusCode int) *http.Response { + return ts.DoRawWithHeaders(verb, path, rawBytes, expectedStatusCode, nil) +} + +func (ts *withServer) DoJSON(verb, path string, params interface{}, expectedStatusCode int, v interface{}) { + resp := ts.Do(verb, path, params, expectedStatusCode) + err := json.NewDecoder(resp.Body).Decode(v) + require.NoError(ts.s.T(), err) + if e, ok := v.(errorer); ok { + require.NoError(ts.s.T(), e.error()) + } +} + +func (ts *withServer) getTestAdminToken() string { + testUser := testUsers["admin1"] + + params := loginRequest{ + Email: testUser.Email, + Password: testUser.PlaintextPassword, + } + j, err := json.Marshal(¶ms) + require.NoError(ts.s.T(), err) + + requestBody := io.NopCloser(bytes.NewBuffer(j)) + resp, err := http.Post(ts.server.URL+"/api/v1/fleet/login", "application/json", requestBody) + require.NoError(ts.s.T(), err) + defer resp.Body.Close() + assert.Equal(ts.s.T(), http.StatusOK, resp.StatusCode) + + var jsn = struct { + User *fleet.User `json:"user"` + Token string `json:"token"` + Err []map[string]string `json:"errors,omitempty"` + }{} + err = json.NewDecoder(resp.Body).Decode(&jsn) + require.Nil(ts.s.T(), err) + + return jsn.Token +} + +func (ts *withServer) applyConfig(spec []byte) { + var appConfigSpec interface{} + err := yaml.Unmarshal(spec, &appConfigSpec) + require.NoError(ts.s.T(), err) + + ts.Do("PATCH", "/api/v1/fleet/config", appConfigSpec, http.StatusOK) +} + +func (ts *withServer) getConfig() *appConfigResponse { + var responseBody *appConfigResponse + ts.DoJSON("GET", "/api/v1/fleet/config", nil, http.StatusOK, &responseBody) + return responseBody +} diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 3b2ab6316a..8d57e4f344 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -2,6 +2,9 @@ package service import ( "context" + "encoding/json" + "io/ioutil" + "net/http" "net/http/httptest" "os" "strings" @@ -9,6 +12,7 @@ import ( eeservice "github.com/fleetdm/fleet/v4/ee/server/service" "github.com/fleetdm/fleet/v4/server/logging" + "github.com/stretchr/testify/assert" "github.com/WatchBeam/clock" "github.com/fleetdm/fleet/v4/server/config" @@ -163,6 +167,9 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...TestServe limitStore, _ := memstore.New(0) r := MakeHandler(svc, config.FleetConfig{}, logger, limitStore) server := httptest.NewServer(r) + t.Cleanup(func() { + server.Close() + }) return users, server } @@ -235,3 +242,21 @@ func testStdoutPluginConfig() config.FleetConfig { c.Osquery.StatusLogPlugin = "stdout" return c } + +func assertBodyContains(t *testing.T, resp *http.Response, expected string) { + bodyBytes, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err) + bodyString := string(bodyBytes) + assert.Contains(t, bodyString, expected) +} + +func getJSON(r *http.Response, target interface{}) error { + return json.NewDecoder(r.Body).Decode(target) +} + +func assertErrorCodeAndMessage(t *testing.T, resp *http.Response, code int, message string) { + err := &fleet.Error{} + require.Nil(t, getJSON(resp, err)) + assert.Equal(t, code, err.Code) + assert.Equal(t, message, err.Message) +} From 6180f26d9d303f8555115738c710afc908ef3e20 Mon Sep 17 00:00:00 2001 From: William Theaker Date: Wed, 15 Sep 2021 19:52:50 -0400 Subject: [PATCH 05/82] Fix typo (#2087) --- .../components/AddPolicyModal/AddPolicyModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/pages/policies/ManagePoliciesPage/components/AddPolicyModal/AddPolicyModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/AddPolicyModal/AddPolicyModal.tsx index 06bbf0a835..0f3e293451 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/AddPolicyModal/AddPolicyModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/AddPolicyModal/AddPolicyModal.tsx @@ -61,7 +61,7 @@ const AddPolicyModal = ({

- To test which hosts return results, it is recommened to first run + To test which hosts return results, it is recommended to first run your query as a live query by heading to Queries and then selecting a query.

From 98499a034950239f89524df3e37c639dc3a2fa43 Mon Sep 17 00:00:00 2001 From: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> Date: Thu, 16 Sep 2021 15:42:22 +0900 Subject: [PATCH 06/82] Docs image replacement (#2073) https://github.com/fleetdm/fleet/pull/2071 (removing hardcoded widths on images) undoes what we previously did for making smaller images look good at <990px breakpoints. Only current examples of these smaller images are on this page in the docs, although there are a couple of instances in the handbook. So I propose that we only crop images that will work at full container width sizes. With that in mind I have replaced one of the affected images on this page. --- docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md b/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md index 2eefc77a57..aa7a894d83 100644 --- a/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md +++ b/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md @@ -95,7 +95,7 @@ To add your own device to Fleet, you'll first need to install the osquery agent. > Take note on where your new Orbit directory is located on you device. Knowing this will be helpful when building the Orbit package in step 3. -Clone Orbit repository +Clone Orbit repository 2. In Fleet UI's Host page, hit the "Add new host" button, and copy your Fleet enroll secret (you'll need this in the next step.) From c4bc41ad4fb8be9e8804ede975c0e89be991e1f5 Mon Sep 17 00:00:00 2001 From: eashaw Date: Thu, 16 Sep 2021 02:23:59 -0500 Subject: [PATCH 07/82] Add page titles and descriptions to fleetdm.com (#2082) * added title to locals in routes * using title and description if they are passed into the locals * fix error being thrown if there is no description * descriptions * fixed error, updated descriptions * update twitter card title * updated descriptions again * updated twitter title --- website/config/routes.js | 78 ++++++++++++++++++++++++++++---- website/views/layouts/layout.ejs | 7 +-- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/website/config/routes.js b/website/config/routes.js index 6a1b6ffdeb..c107ee8700 100644 --- a/website/config/routes.js +++ b/website/config/routes.js @@ -13,19 +13,77 @@ module.exports.routes = { // ╦ ╦╔═╗╔╗ ╔═╗╔═╗╔═╗╔═╗╔═╗ // ║║║║╣ ╠╩╗╠═╝╠═╣║ ╦║╣ ╚═╗ // ╚╩╝╚═╝╚═╝╩ ╩ ╩╚═╝╚═╝╚═╝ - 'GET /': { action: 'view-homepage-or-redirect', locals: { isHomepage: true } }, - 'GET /company/contact': { action: 'view-contact' }, - 'GET /get-started': { action: 'view-get-started' }, - 'GET /pricing': { action: 'view-pricing' }, - 'GET /press-kit': { action: 'view-press-kit' }, + 'GET /': { + action: 'view-homepage-or-redirect', + locals: { isHomepage: true } + }, - 'GET /queries': { action: 'view-query-library' }, - 'GET /queries/:slug': { action: 'view-query-detail' }, + 'GET /company/contact': { + action: 'view-contact', + locals:{ + title: 'Contact us | Fleet for osquery', + description: 'Get in touch with our team.' + } + }, - 'GET /docs/?*': { skipAssets: false, action: 'docs/view-basic-documentation' },// handles /docs and /docs/foo/bar - // 'GET /handbook/?*': { skipAssets: false, action: 'handbook/view-basic-handbook' },// handles /handbook and /handbook/foo/bar + 'GET /get-started': { + action: 'view-get-started' , + locals:{ + title: 'Get Started | Fleet for osquery', + description: 'Learn about getting started with Fleet.' + } + }, - 'GET /transparency': { action: 'view-transparency' }, + 'GET /pricing': { + action: 'view-pricing', + locals:{ + title: 'Pricing | Fleet for osquery', + description: 'View Fleet plans and pricing details.' + } + }, + + 'GET /press-kit': { + action: 'view-press-kit', + locals:{ + title: 'Press kit | Fleet for osquery', + description: 'Download Fleet logos, wallpapers, and screenshots.' + } + }, + + 'GET /queries': { + action: 'view-query-library', + locals:{ + title: 'Queries | Fleet for osquery', + description: 'A growing collection of useful queries for organizations deploying Fleet and osquery.' + } + }, + + 'GET /queries/:slug': { + action: 'view-query-detail', + locals:{ + title: 'Query details | Fleet for osquery', + description: 'View more information about a query in Fleet\'s standard query library', + } + }, + + 'GET /docs/?*': { + skipAssets: false, + action: 'docs/view-basic-documentation', + locals:{ + title: 'Documentation | Fleet for osquery', + description: 'Documentation for Fleet for osquery.', + } + },// handles /docs and /docs/foo/bar + + // 'GET /handbook/?*': { skipAssets: false, action: 'handbook/view-basic-handbook' },// handles /handbook and /handbook/foo/bar + + 'GET /transparency': { + action: 'view-transparency', + locals:{ + title: 'Transparency | Fleet for osquery', + description: 'Learn what data osquery can see.', + } + }, diff --git a/website/views/layouts/layout.ejs b/website/views/layouts/layout.ejs index 2f956ccbbd..d609febe48 100644 --- a/website/views/layouts/layout.ejs +++ b/website/views/layouts/layout.ejs @@ -7,14 +7,15 @@ %> - Fleet for osquery | Open source device management + <%= typeof title !== 'undefined' ? title : 'Fleet for osquery | Open source device management' %> + <% /* Viewport tag for sensible mobile support */ %> - - + + <% /* Script tags should normally be included further down the page- but any scripts that load fonts (e.g. Fontawesome ≥v5) are special exceptions to the From 57b063125e354af8a7203a91d1c42bd7853fdbbb Mon Sep 17 00:00:00 2001 From: eashaw Date: Thu, 16 Sep 2021 02:45:14 -0500 Subject: [PATCH 08/82] json highlighting (#2084) --- docs/1-Using-Fleet/3-REST-API.md | 346 +++++++++++++++---------------- 1 file changed, 173 insertions(+), 173 deletions(-) diff --git a/docs/1-Using-Fleet/3-REST-API.md b/docs/1-Using-Fleet/3-REST-API.md index b055201859..d0a59d4478 100644 --- a/docs/1-Using-Fleet/3-REST-API.md +++ b/docs/1-Using-Fleet/3-REST-API.md @@ -100,7 +100,7 @@ Authenticates the user with the specified credentials. Use the token returned fr ##### Request body -``` +```json { "email": "janedoe@example.com", "password": "VArCjNW7CfsxGp67" @@ -166,7 +166,7 @@ Sends a password reset email to the specified email. Requires that SMTP is confi ##### Request body -``` +```json { "email": "janedoe@example.com" } @@ -180,7 +180,7 @@ Sends a password reset email to the specified email. Requires that SMTP is confi `Status: 500` -``` +```json { "message": "Unknown Error", "errors": [ @@ -213,7 +213,7 @@ Changes the password for the authenticated user. ##### Request body -``` +```json { "old_password": "VArCjNW7CfsxGp67", "new_password": "zGq7mCLA6z4PzArC" @@ -228,7 +228,7 @@ Changes the password for the authenticated user. `Status: 422 Unprocessable entity` -``` +```json { "message": "Validation Failed", "errors": [ @@ -260,7 +260,7 @@ Resets a user's password. Which user is determined by the password reset token u ##### Request body -``` +```json { "new_password": "abc123" "new_password_confirmation": "abc123" @@ -272,7 +272,7 @@ Resets a user's password. Which user is determined by the password reset token u `Status: 200` -``` +```json {} ``` @@ -292,7 +292,7 @@ Retrieves the user data for the authenticated user. `Status: 200` -``` +```json { "user": { "created_at": "2020-11-13T22:57:12Z", @@ -324,7 +324,7 @@ Resets the password of the authenticated user. Requires that `force_password_res ##### Request body -``` +```json { "new_password": "sdPz8CV5YhzH47nK" } @@ -334,7 +334,7 @@ Resets the password of the authenticated user. Requires that `force_password_res `Status: 200` -``` +```json { "user": { "created_at": "2020-11-13T22:57:12Z", @@ -368,7 +368,7 @@ Gets the current SSO configuration. `Status: 200` -``` +```json { "settings": { "idp_name": "IDP Vendor 1", @@ -396,7 +396,7 @@ Gets the current SSO configuration. ##### Request body -``` +```json { "relay_url": "/hosts/manage" } @@ -410,7 +410,7 @@ Gets the current SSO configuration. `Status: 500` -``` +```json { "message": "Unknown Error", "errors": [ @@ -440,7 +440,7 @@ This is the callback endpoint that the identity provider will use to send securi ##### Request body -``` +```json { "SAMLResponse": "" } @@ -450,7 +450,7 @@ This is the callback endpoint that the identity provider will use to send securi `Status: 200` -``` +```json {} ``` @@ -494,7 +494,7 @@ If `additional_info_filters` is not specified, no `additional` information will ##### Request query parameters -``` +```json { "page": 0, "per_page": 100, @@ -506,7 +506,7 @@ If `additional_info_filters` is not specified, no `additional` information will `Status: 200` -``` +```json { "hosts": [ { @@ -570,7 +570,7 @@ None. `Status: 200` -``` +```json { "online_count": 2267, "offline_count": 141, @@ -601,7 +601,7 @@ The endpoint returns the host's installed `software` if the software inventory f `Status: 200` -``` +```json { "host": { "created_at": "2021-08-19T02:02:22Z", @@ -738,7 +738,7 @@ Returns the information of the host specified using the `uuid`, `osquery_host_id `Status: 200` -``` +```json { "host": { "created_at": "2020-11-05T05:09:44Z", @@ -804,7 +804,7 @@ Deletes the specified host from Fleet. Note that a deleted host will fail authen `Status: 200` -``` +```json {} ``` @@ -828,7 +828,7 @@ Flags the host details to be refetched the next time the host checks in for live `Status: 200` -``` +```json {} ``` @@ -851,7 +851,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "team_id": 1, "hosts": [3, 2, 4, 6, 1, 5, 7] @@ -862,7 +862,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json {} ``` @@ -885,7 +885,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "team_id": 1, "filters": { @@ -898,7 +898,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json {} ``` @@ -938,7 +938,7 @@ Creates a dynamic label. ##### Request body -``` +```json { "name": "Ubuntu hosts", "description": "Filters ubuntu hosts", @@ -951,7 +951,7 @@ Creates a dynamic label. `Status: 200` -``` +```json { "label": { "created_at": "0001-01-01T00:00:00Z", @@ -989,7 +989,7 @@ Modifies the specified label. Note: Label queries and platforms are immutable. T ##### Request body -``` +```json { "name": "macOS label", "description": "Now this label only includes macOS machines", @@ -1001,7 +1001,7 @@ Modifies the specified label. Note: Label queries and platforms are immutable. T `Status: 200` -``` +```json { "label": { "created_at": "0001-01-01T00:00:00Z", @@ -1040,7 +1040,7 @@ Returns the specified label. `Status: 200` -``` +```json { "label": { "created_at": "2021-02-09T22:09:43Z", @@ -1080,7 +1080,7 @@ Returns a list of all the labels in Fleet. `Status: 200` -``` +```json { "labels": [ { @@ -1184,7 +1184,7 @@ Returns a list of the hosts that belong to the specified label. `Status: 200` -``` +```json { "hosts": [ { @@ -1251,7 +1251,7 @@ Deletes the label specified by name. `Status: 200` -``` +```json {} ``` @@ -1275,7 +1275,7 @@ Deletes the label specified by ID. `Status: 200` -``` +```json {} ``` @@ -1301,7 +1301,7 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als ##### Request body -``` +```json { "specs": [ { @@ -1326,7 +1326,7 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als `Status: 200` -``` +```json {} ``` @@ -1346,7 +1346,7 @@ None. `Status: 200` -``` +```json { "specs": [ { @@ -1421,7 +1421,7 @@ None. `Status: 200` -``` +```json { "specs": { "id": 12, @@ -1480,7 +1480,7 @@ None. `Status: 200` -``` +```json { "users": [ { @@ -1512,7 +1512,7 @@ None. `Status: 401 Authentication Failed` -``` +```json { "message": "Authentication Failed", "errors": [ @@ -1548,7 +1548,7 @@ Creates a user account after an invited user provides registration information a ##### Request query parameters -``` +```json { "email": "janedoe@example.com", "invite_token": "SjdReDNuZW5jd3dCbTJtQTQ5WjJTc2txWWlEcGpiM3c=", @@ -1572,7 +1572,7 @@ Creates a user account after an invited user provides registration information a `Status: 200` -``` +```json { "user": { "created_at": "0001-01-01T00:00:00Z", @@ -1594,7 +1594,7 @@ Creates a user account after an invited user provides registration information a `Status: 401 Authentication Failed` -``` +```json { "message": "Authentication Failed", "errors": [ @@ -1610,7 +1610,7 @@ Creates a user account after an invited user provides registration information a `Status: 404 Resource Not Found` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -1628,7 +1628,7 @@ Creates a user account after an invited user provides registration information a The same error will be returned whenever one of the required parameters fails the validation. -``` +```json { "message": "Validation Failed", "errors": [ @@ -1664,7 +1664,7 @@ Creates a user account without requiring an invitation, the user is enabled imme ##### Request body -``` +```json { "name": "Jane Doe", "email": "janedoe@example.com", @@ -1686,7 +1686,7 @@ Creates a user account without requiring an invitation, the user is enabled imme `Status: 200` -``` +```json { "user": { "created_at": "0001-01-01T00:00:00Z", @@ -1718,7 +1718,7 @@ Creates a user account without requiring an invitation, the user is enabled imme `Status: 404 Resource Not Found` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -1748,7 +1748,7 @@ Returns all information about a specific user. ##### Request query parameters -``` +```json { "id": 1 } @@ -1758,7 +1758,7 @@ Returns all information about a specific user. `Status: 200` -``` +```json { "user": { "created_at": "2020-12-10T05:20:25Z", @@ -1780,7 +1780,7 @@ Returns all information about a specific user. `Status: 404 Resource Not Found` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -1815,7 +1815,7 @@ Returns all information about a specific user. ##### Request body -``` +```json { "name": "Jane Doe", "global_role": "admin" @@ -1826,7 +1826,7 @@ Returns all information about a specific user. `Status: 200` -``` +```json { "user": { "created_at": "2021-02-03T16:11:06Z", @@ -1850,7 +1850,7 @@ Returns all information about a specific user. ##### Request body -``` +```json { "teams": [ { @@ -1869,7 +1869,7 @@ Returns all information about a specific user. `Status: 200` -``` +```json { "user": { "created_at": "2021-02-03T16:11:06Z", @@ -1916,7 +1916,7 @@ Delete the specified user from Fleet. `Status: 200` -``` +```json {} ``` @@ -1939,7 +1939,7 @@ The selected user is logged out of Fleet and required to reset their password du ##### Request body -``` +```json { "require": true } @@ -1949,7 +1949,7 @@ The selected user is logged out of Fleet and required to reset their password du `Status: 200` -``` +```json { "user": { "created_at": "2021-02-23T22:23:34Z", @@ -1984,7 +1984,7 @@ None. `Status: 200` -``` +```json { "sessions": [ { @@ -2026,7 +2026,7 @@ Deletes the selected user's sessions in Fleet. Also deletes the user's API token `Status: 200` -``` +```json {} ``` @@ -2057,7 +2057,7 @@ Returns the session information for the session specified by ID. `Status: 200` -``` +```json { "session_id": 1, "user_id": 1, @@ -2085,7 +2085,7 @@ Deletes the session specified by ID. When the user associated with the session n `Status: 200` -``` +```json {} ``` @@ -2130,7 +2130,7 @@ Returns the query specified by ID. `Status: 200` -``` +```json { "query": { "created_at": "2021-01-19T17:08:24Z", @@ -2179,7 +2179,7 @@ Returns a list of all queries in the Fleet instance. `Status: 200` -``` +```json { "queries": [ { @@ -2250,7 +2250,7 @@ Returns a list of all queries in the Fleet instance. ##### Request body -``` +```json { "description": "This is a new query.", "name": "new_query", @@ -2262,7 +2262,7 @@ Returns a list of all queries in the Fleet instance. `Status: 200` -``` +```json { "query": { "created_at": "0001-01-01T00:00:00Z", @@ -2302,7 +2302,7 @@ Returns the query specified by ID. ##### Request body -``` +```json { "name": "new_title_for_my_query" } @@ -2312,7 +2312,7 @@ Returns the query specified by ID. `Status: 200` -``` +```json { "query": { "created_at": "2021-01-22T17:23:27Z", @@ -2350,7 +2350,7 @@ Deletes the query specified by name. `Status: 200` -``` +```json {} ``` @@ -2374,7 +2374,7 @@ Deletes the query specified by ID. `Status: 200` -``` +```json {} ``` @@ -2396,7 +2396,7 @@ Deletes the queries specified by ID. Returns the count of queries successfully d ##### Request body -``` +```json { "ids": [ 2, 24, 25 @@ -2408,7 +2408,7 @@ Deletes the queries specified by ID. Returns the count of queries successfully d `Status: 200` -``` +```json { "deleted": 3 } @@ -2432,7 +2432,7 @@ None. `Status: 200` -``` +```json { "specs": [ { @@ -2469,7 +2469,7 @@ Returns the name, description, and SQL of the query specified by name. `Status: 200` -``` +```json { "specs": { "name": "query1", @@ -2497,7 +2497,7 @@ Creates and/or modifies the queries included in the specs list. To modify an exi ##### Request body -``` +```json { "specs": [ { @@ -2518,7 +2518,7 @@ Creates and/or modifies the queries included in the specs list. To modify an exi `Status: 200` -``` +```json {} ``` @@ -2540,7 +2540,7 @@ None. `Status: 200` -``` +```json {} ``` @@ -2562,7 +2562,7 @@ None. `Status: 200` -``` +```json {} ``` @@ -2590,7 +2590,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query": "select instance_id from system_info", "selected": { @@ -2603,7 +2603,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2629,7 +2629,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query": "select instance_id from system_info;", "selected": { @@ -2642,7 +2642,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2686,7 +2686,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query_id": 1, "selected": { @@ -2701,7 +2701,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2727,7 +2727,7 @@ One of `query` and `query_id` must be specified. ##### Request body -``` +```json { "query": "select instance_id from system_info", "selected": { @@ -2742,7 +2742,7 @@ One of `query` and `query_id` must be specified. `Status: 200` -``` +```json { "campaign": { "created_at": "0001-01-01T00:00:00Z", @@ -2806,7 +2806,7 @@ socket.onmessage = ({ data }) => { ###### Response data -``` +```json o ``` @@ -2814,7 +2814,7 @@ o ###### Request data -``` +```json [ { "type": "auth", @@ -2823,7 +2823,7 @@ o ] ``` -``` +```json [ { "type": "select_campaign", @@ -2836,7 +2836,7 @@ o ###### Response data -``` +```json // Sends the total number of hosts targeted and segments them by status [ @@ -2853,7 +2853,7 @@ o ] ``` -``` +```json // Sends the expected results, actual results so far, and the status of the live query [ @@ -2869,7 +2869,7 @@ o ] ``` -``` +```json // Sends the result for a given host [ @@ -2889,7 +2889,7 @@ o ] ``` -``` +```json // Sends the status of "finished" when messages with the results for all expected hosts have been sent [ @@ -2948,7 +2948,7 @@ socket.onmessage = ({ data }) => { ###### Response data -``` +```json o ``` @@ -2956,7 +2956,7 @@ o ###### Request data -``` +```json [ { "type": "auth", @@ -2965,7 +2965,7 @@ o ] ``` -``` +```json [ { "type": "select_campaign", @@ -2974,11 +2974,11 @@ o ] ``` -##### `socket.onmessage()` +##### socket.onmessage() ###### Response data -``` +```json // Sends the total number of hosts targeted and segments them by status [ @@ -2995,7 +2995,7 @@ o ] ``` -``` +```json // Sends the expected results, actual results so far, and the status of the live query [ @@ -3011,7 +3011,7 @@ o ] ``` -``` +```json // Sends the result for a given host [ @@ -3031,7 +3031,7 @@ o ] ``` -``` +```json // Sends the status of "finished" when messages with the results for all expected hosts have been sent [ @@ -3078,7 +3078,7 @@ None. `Status: 200` -``` +```json { "global_schedule": [ { @@ -3141,7 +3141,7 @@ None. ##### Request body -``` +```json { "interval": 86400, "query_id": 2, @@ -3153,7 +3153,7 @@ None. `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3199,7 +3199,7 @@ None. ##### Request body -``` +```json { "interval": 604800, } @@ -3209,7 +3209,7 @@ None. `Status: 200` -``` +```json { "scheduled": { "created_at": "2021-07-16T14:40:15Z", @@ -3246,7 +3246,7 @@ None. `Status: 200` -``` +```json {} ``` @@ -3285,7 +3285,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": [ { @@ -3349,7 +3349,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request body -``` +```json { "interval": 86400, "query_id": 2, @@ -3361,7 +3361,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3404,7 +3404,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request body -``` +```json { "interval": 604800, } @@ -3414,7 +3414,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "2021-07-16T14:40:15Z", @@ -3454,7 +3454,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json {} ``` @@ -3497,7 +3497,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request query parameters -``` +```json { "description": "Collects osquery data.", "host_ids": [], @@ -3510,7 +3510,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "pack": { "created_at": "0001-01-01T00:00:00Z", @@ -3550,7 +3550,7 @@ This allows you to easily configure scheduled queries that will impact a whole t ##### Request query parameters -``` +```json { "description": "MacOS hosts are targeted", "host_ids": [], @@ -3562,7 +3562,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "pack": { "created_at": "2021-01-25T22:32:45Z", @@ -3599,7 +3599,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "pack": { "created_at": "2021-01-25T22:32:45Z", @@ -3637,7 +3637,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "packs": [ { @@ -3690,7 +3690,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json {} ``` @@ -3712,7 +3712,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json {} ``` @@ -3734,7 +3734,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": [ { @@ -3815,7 +3815,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Request body -``` +```json { "interval": 120, "pack_id": 15, @@ -3832,7 +3832,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3871,7 +3871,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "0001-01-01T00:00:00Z", @@ -3915,7 +3915,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Request body -``` +```json { "platform": "", } @@ -3925,7 +3925,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json { "scheduled": { "created_at": "2021-01-28T19:40:04Z", @@ -3964,7 +3964,7 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -``` +```json {} ``` @@ -3982,7 +3982,7 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` -``` +```json { "specs": [ { @@ -4102,7 +4102,7 @@ Returns the specs for all packs in the Fleet instance. ##### Request body -``` +```json { "specs": [ { @@ -4193,7 +4193,7 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` -``` +```json {} ``` @@ -4217,7 +4217,7 @@ Returns the spec for the specified pack by pack name. `Status: 200` -``` +```json { "specs": { "id": 15, @@ -4316,7 +4316,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "policies": [ { @@ -4355,7 +4355,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "policy": { "id": 1, @@ -4383,7 +4383,7 @@ Hosts that do not return results for a policy's query are "Failing." #### Request body -``` +```json { "query_id": 12 } @@ -4393,7 +4393,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "policy": { "id": 2, @@ -4421,7 +4421,7 @@ Hosts that do not return results for a policy's query are "Failing." #### Request body -``` +```json { "ids": [ 1 ] } @@ -4431,7 +4431,7 @@ Hosts that do not return results for a policy's query are "Failing." `Status: 200` -``` +```json { "deleted": 1 } @@ -4474,7 +4474,7 @@ Returns a list of the activities that have been performed in Fleet. The followin ##### Default response -``` +```json { "activities": [ { @@ -4637,7 +4637,7 @@ The returned lists are filtered based on the hosts the requesting user has acces ##### Request body -``` +```json { "query": "172", "selected": { @@ -4650,7 +4650,7 @@ The returned lists are filtered based on the hosts the requesting user has acces ##### Default response -``` +```json { "targets": { "hosts": [ @@ -4800,7 +4800,7 @@ None. `Status: 200` -``` +```json { "certificate_chain": } @@ -4824,7 +4824,7 @@ None. `Status: 200` -``` +```json { "org_info": { "org_name": "fleet", @@ -5005,7 +5005,7 @@ Modifies the Fleet's configuration with the supplied information. ##### Request body -``` +```json { "org_info": { "org_name": "Fleet Device Management", @@ -5023,7 +5023,7 @@ Modifies the Fleet's configuration with the supplied information. `Status: 200` -``` +```json { "org_info": { "org_name": "Fleet Device Management", @@ -5148,7 +5148,7 @@ None. `Status: 200` -``` +```json { "spec": { "secrets": [ @@ -5181,7 +5181,7 @@ Replaces the active global enroll secrets with the secrets specified. ##### Request body -``` +```json { "spec": { "secrets": [ @@ -5199,7 +5199,7 @@ Replaces the active global enroll secrets with the secrets specified. `Status: 200` -``` +```json {} ``` @@ -5221,7 +5221,7 @@ None. `Status: 200` -``` +```json { "secrets": [ { @@ -5251,7 +5251,7 @@ None. ##### Request body -``` +```json { "email": "john_appleseed@example.com", "name": John, @@ -5276,7 +5276,7 @@ None. `Status: 200` -``` +```json { "invite": { "created_at": "0001-01-01T00:00:00Z", @@ -5334,7 +5334,7 @@ Returns a list of the active invitations in Fleet. `Status: 200` -``` +```json { "invites": [ { @@ -5381,7 +5381,7 @@ Delete the specified invite from Fleet. `Status: 200` -``` +```json {} ``` @@ -5405,7 +5405,7 @@ Verify the specified invite. `Status: 200` -``` +```json { "invite": { "created_at": "2021-01-15T00:58:33Z", @@ -5424,7 +5424,7 @@ Verify the specified invite. `Status: 404` -``` +```json { "message": "Resource Not Found", "errors": [ @@ -5454,7 +5454,7 @@ None. `Status: 200` -``` +```json { "version": "3.9.0-93-g1b67826f-dirty", "branch": "version", @@ -5497,7 +5497,7 @@ None. `Status: 200` -``` +```json { "carves": [ { @@ -5552,7 +5552,7 @@ Retrieves the specified carve. `Status: 200` -``` +```json { "carve": { "id": 1, @@ -5592,7 +5592,7 @@ Retrieves the specified carve block. This endpoint retrieves the data that was c `Status: 200` -``` +```json { "data": "aG9zdHMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA..." } @@ -5626,7 +5626,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "teams: [ { @@ -5724,7 +5724,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "name": "workstations" } @@ -5734,7 +5734,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "teams: [ { @@ -5793,7 +5793,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "user_ids": [1, 17, 22, 32], } @@ -5803,7 +5803,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "team": { "name": "Workstations", @@ -5845,7 +5845,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], } @@ -5855,7 +5855,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "team": { "name": "Workstations", @@ -5897,7 +5897,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "agent_options": { "spec": { @@ -5929,7 +5929,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "team": { "name": "Workstations", @@ -5985,7 +5985,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json {} ``` @@ -6009,7 +6009,7 @@ _Available in Fleet Premium_ ##### Request body -``` +```json { "list": [ { @@ -6044,7 +6044,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { "list": [ { @@ -6104,7 +6104,7 @@ _Available in Fleet Premium_ `Status: 200` -``` +```json { “software”: [ { From 57ba22d2b08e57e1ac03152397e58f37c62be2b4 Mon Sep 17 00:00:00 2001 From: eashaw Date: Thu, 16 Sep 2021 03:04:26 -0500 Subject: [PATCH 09/82] Add clickable anchor links in fleetdm.com/docs (#2010) * added a default renderer for headings to keep the links consistent with the ids * adjusted the render function to create a link for each heading * added styles for the heading links and link icon * changing variable names to be more specific, fixing sidebar links, hiding autogenerated
    s * Removed lodash require, updated comment, removed px from icon filename, and updated link to image. --- website/api/helpers/strings/to-html.js | 10 ++++++++-- website/assets/images/icon-link-16x16@2x.png | Bin 0 -> 772 bytes .../js/pages/docs/basic-documentation.page.js | 6 ++++-- .../styles/pages/docs/basic-documentation.less | 13 ++++++++++++- 4 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 website/assets/images/icon-link-16x16@2x.png diff --git a/website/api/helpers/strings/to-html.js b/website/api/helpers/strings/to-html.js index 09bfabc6df..0ec855ce7f 100644 --- a/website/api/helpers/strings/to-html.js +++ b/website/api/helpers/strings/to-html.js @@ -73,8 +73,14 @@ module.exports = { smartLists: true, smartypants: false, }; - - if (inputs.addIdsToHeadings === false) { + if (inputs.addIdsToHeadings === true) { + var headingRenderer = new marked.Renderer(); + headingRenderer.heading = function (text, level) { + var headingID = _.kebabCase(text); + return ''+text+''; + }; + markedOpts.renderer = headingRenderer; + } else { var renderer = new marked.Renderer(); renderer.heading = function (text, level) { return ''+text+''; diff --git a/website/assets/images/icon-link-16x16@2x.png b/website/assets/images/icon-link-16x16@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..41120a4844daa9280aef84f423fa9dc503dbc24a GIT binary patch literal 772 zcmV+f1N;1mP)t6voqJM_%=%}oC-!aqRtdU_sE@Pe7EfKUO32_lZ!&8`O@OT$%LgaQ#o90m+m z3|!w~mN@+VERN5TLIo%}F@KStp}~A9f}j9+k}JgiRPB#Hfvx0n??BvV$77)h#RuL! zuyO9?Q37`CeC?jM*)qWB;4APUSHv+U5*7W`G1-0fAOXJQ_u%|^<-ik|hN3=^&nUZV z6S)Gz+`y!eFnrsbHkfY5yT~`4nr^^LK=kUiPE@MNOg({6hBUj~tI#z}#6( z?+rd+*E)Aon4%APIW15Aobr$L1#radz5>z{5!T2W5F`1T0E_zeIal%m_F!*zBrCZ- z^oA%7$XzIid?~=^-Ue~dVdD4t|F%Un#ahTSTWugol8A+T740@x(sb4!4r&M-GHAJV zH%v}y!pR7XC}rKNwgw*LdS&U)X&8Sx8Oq}jjzDSAIxnG3LwsqdYC#aTnr_~1OIdX| z?fdL}DIgy { - let subtopics = $('#body-content').find('h2').map((_, el) => el.innerHTML); + let subtopics = $('#body-content').find('h2').map((_, el) => el.innerText); subtopics = $.makeArray(subtopics).map((title) => { + // Removing all apostrophes from the title to keep _.kebabCase() from turning words like 'user’s' into 'user-s' + let kebabCaseFriendlyTitle = title.replace(/[\’]/g, ''); return { title, - url: '#' + _.kebabCase(title), + url: '#' + _.kebabCase(kebabCaseFriendlyTitle), }; }); return subtopics; diff --git a/website/assets/styles/pages/docs/basic-documentation.less b/website/assets/styles/pages/docs/basic-documentation.less index 6f83ce2d41..b1394993c1 100644 --- a/website/assets/styles/pages/docs/basic-documentation.less +++ b/website/assets/styles/pages/docs/basic-documentation.less @@ -38,6 +38,17 @@ } } + .docs-heading:hover { + + .docs-link { + height: 16px; + vertical-align: middle; + margin-left: 8px; + content: url('/images/icon-link-16x16@2x.png'); + } + + } + [purpose='search'] { @@ -341,7 +352,7 @@ padding-bottom: 24px; } - h1 + ul { + span + ul { display: none; // Hides links at top of some markdown files } From 74e3f02e6c4e86c0cf2fa342589d1e89692af002 Mon Sep 17 00:00:00 2001 From: Martavis Parker <47053705+martavis@users.noreply.github.com> Date: Thu, 16 Sep 2021 10:32:33 -0700 Subject: [PATCH 10/82] Docs for moving forward in frontend (#2070) * Created docs for moving forward in frontend * edited doc for clarity * typos * fixed route to page --- frontend/README.md | 293 ++++++++++++++++++++++++++++------ frontend/README_deprecated.md | 128 +++++++++++++++ 2 files changed, 376 insertions(+), 45 deletions(-) create mode 100644 frontend/README_deprecated.md diff --git a/frontend/README.md b/frontend/README.md index 6331e7ee9f..e7770abd47 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,6 +1,18 @@ # Fleet Front-End -The Fleet front-end is a Single Page Application using React and Redux. +The Fleet front-end is a Single Page Application using React with Typescript and Hooks. + +## Table of Contents +- [Running the Fleet web app](#running-the-fleet-web-app) +- [Directory Structure](#directory-structure) +- [Deprecated](#deprecated) +- [Patterns](#patterns) + - [Typing](#typing) + - [React Hooks (Functional Components)](#react-hooks-functional-components) + - [React Context](#react-context) + - [Fleet API Calls](#fleet-api-calls) + - [Page Routing](#page-routing) + - [Other](#other) ## Running the Fleet web app @@ -10,25 +22,19 @@ consult the [Contributing documentation](../docs/3-Contributing/README.md). ## Directory Structure Component directories in the Fleet front-end application encapsulate the entire -component, including files for the component, helper functions, styles, and tests. The +component, including files for the component and its styles. The typical directory structure for a component is as follows: ``` -|-- ComponentName -| |-- _styles.scss -| |-- ComponentName.jsx -| |-- ComponentName.tests.jsx -| |-- helpers.js -| |-- helpers.tests.js -| |-- index.js +└── ComponentName + ├── _styles.scss + ├── ComponentName.tsx + ├── index.ts ``` - `_styles.scss`: The component css styles -- `ComponentName.jsx`: The React component -- `ComponentName.tests.jsx`: The React component tests -- `helpers.js`: Helper functions used by the component -- `helpers.tests.js`: Tests for the component's helper functions -- `index.js`: Exports the React component +- `ComponentName.tsx`: The React component +- `index.ts`: Exports the React component - This file is helpful as it allows other components to import the component by it's directory name. Without this file the component name would have to be duplicated during imports (`components/ComponentName` vs. `components/ComponentName/ComponentName`). @@ -46,40 +52,34 @@ The component directory contains the React components rendered by pages. They are typically not connected to the redux state but receive props from their parent components to render data and handle user interactions. +### [context](./context) + +The context directory contains the React Context API pattern for various entities. +Only entities that are needed across the app has a global context. For example, +the [logged in user](./context/app.tsx) (`currentUser`) has multiple pages and components +where its information is pulled. + ### [interfaces](./interfaces) -Files in the interfaces directory are used to specify the PropTypes for a reusable Fleet +Files in the interfaces directory are used to specify the Typescript interface for a reusable Fleet entity. This is designed to DRY up the code and increase re-usability. These -interfaces are imported into component files and implemented when defining the -component's PropTypes. +interfaces are imported in to component files and implemented when defining the +component's props. -### [fleet](./fleet) - -The default export of the `fleet` directory is the API client. More info can be -found at the [API client documentation page](./fleet/README.md). +**Additionally, local interfaces are used for props of local components.** ### [layouts](https://github.com/fleetdm/fleet/tree/main/frontend/layouts) The Fleet application has only 1 layout, the [Core Layout](./layouts/CoreLayout/CoreLayout.jsx). -The Layout is rendered from the [router](./router/index.jsx) and are used to set up the general app UI (header, sidebar) and render child components. +The Layout is rendered from the [router](./router/index.tsx) and are used to set up the general +app UI (header, sidebar) and render child components. The child components rendered by the layout are typically page components. ### [pages](./pages) Page components are React components typically rendered from the [router](./router). -These components are connected to redux state and are used to gather data from -redux and pass that data to child components (located in the [components -directory](./components). As -connected components, Pages are also used to dispatch actions. Actions -dispatched from Pages are intended to update redux state and oftentimes include -making a call to the Fleet API. - -### [redux](./redux) - -The redux directory holds all of the application's redux middleware, actions, -and reducers. The redux directory also creates the [store](./redux/store.js) which is used in the router. -More information about the redux configuration can be found at the [Redux -Documentation page](./redux/README.md) +React Router passed props to these pages in case they are needed. Examples include +the `router`, `location`, and `params` objects. ### [router](./router) @@ -90,6 +90,10 @@ file which holds the application paths as string constants for reference throughout the app. These paths are typically referenced from the [App Constants](./app_constants) object. +### [services](./services) + +CRUD functions for all Fleet entities (e.g. `query`) that link directly to the Fleet API. + ### [styles](./styles) The styles directory contains the general app style setup and variables. It @@ -100,22 +104,221 @@ includes variables for the app color hex codes, fonts (families, weights and siz The templates directory contains the HTML file that renders the React application via including the `bundle.js` and `bundle.css` files. The HTML page also includes the HTML element in which the React application is mounted. -### [test](./test) - -The test directory includes test helpers, API request mocks, and stubbed data entities for use in test files. -More on test helpers, stubs, and request mocks [here](./test/README.md). - ### [utilities](./utilities) -The utilities directory contains re-usable functions for use throughout the +The utilities directory contains re-usable functions and constants for use throughout the application. The functions include helpers to convert an array of objects to CSV, debounce functions to prevent multiple form submissions, format API errors, etc. -## Forms +## Deprecated -For details on creating a Fleet form visit the [Fleet Form Documentation](./components/forms/README.md). +These directories and files are still used (as of 9/14/21) but are being replaced by newer code: -## API Client +- [fleet](./fleet), now using [services](./services) +- [redux](./redux), now using [services](./services), local states, and various entities directly (e.g. React Router) +- [Form.jsx Higher Order Component](./components/forms/README.md), now creating forms with local states with React Hooks (i.e. `useState`) -For details on the Fleet API Client visit the [Fleet API Client Documentation](./fleet/README.md). +To view the deprecated documentation, [click here](./README_deprecated.md). + +## Patterns + +### Typing +All Javascript and React files use Typescript, meaning the extensions are `.ts` and `.tsx`. Here are the guidelines on how we type at Fleet: + +- Use *[global entity interfaces](#interfaces)* when interfaces are used multiple times across the app +- Use *local interfaces* when typing entities limited to the specific page or component +- Local interfaces for page and component props + + ```typescript + // page + interface IPageProps { + prop1: string; + prop2: number; + ... + } + + // Note: Destructure props in page/component signature + const PageOrComponent = ({ + prop1, + prop2, + }: IPageProps) => { + + return ( + // ... + ); + }; + ``` + +- Local states +```typescript +const [item, setItem] = useState(""); +``` + +- Fetch function signatures (i.e. `react-query`) +```typescript +useQuery(params) +``` + +- Custom functions, including callbacks +```typescript +const functionWithTableName = (tableName: string): boolean => { + // do something +}; +``` + +### React Hooks (Functional Components) + +[Hooks](https://reactjs.org/docs/hooks-intro.html) are used to track state and use other features +of React. Hooks are only allowed in functional components, which are created like so: + +```typescript +import React, { useState, useEffect } from "React"; + +const PageOrComponent = (props) => { + const [item, setItem] = useState(""); + + // runs only on first mount (replaces componentDidMount) + useEffect(() => { + // do something + }, []); + + // runs only when `item` changes (replaces componentDidUpdate) + useEffect(() => { + // do something + }, [item]); + + return ( + // ... + ); +}; +``` + +**Note: Other hooks are available per [React's documentation](https://reactjs.org/docs/hooks-intro.html).** + +### React Context + +[React Context](https://reactjs.org/docs/context.html) is a store similar to Redux. It stores +data that is desired and allows for retrieval of that data in whatever component is in need. +View currently working contexts in the [context directory](./context). + +### Fleet API Calls + +**Deprecated:** + +Redux was used to make API calls, along with the [fleet](./fleet) directory. + +**Current:** + +The [services](./services) directory stores all API calls and is to be used in two ways: +- A direct `async/await` assignment +- Using `react-query` if requirements call for loading data right away or based on dependencies. + +Examples below: + +*Direct assignment* +```typescript +// page +import ... +import queryAPI from "services/entities/queries"; + +const PageOrComponent = (props) => { + const doSomething = async () => { + const response = await queryAPI.load(param); + // do something + }; + + return ( + // ... + ); +}; +``` + +*React Query* + +`react-query` ([docs here](https://react-query.tanstack.com/overview)) is a data-fetching library that +gives us the ability to fetch, cache, sync and update data with a myriad of options and properties. + +```typescript +import ... +import { useQuery, useMutation } from "react-query"; +import queryAPI from "services/entities/queries"; + +const PageOrComponent = (props) => { + // retrieve the query based on page/component load + // and dependencies for when to refetch + const { + isLoading, + data, + error, + ...otherProps, + } = useQuery( + "query", + () => queryAPI.load(param), + { + ...options + } + ); + + // `props` is a bucket of properties that can be used when + // updating data. for example, if you need to know whether + // a mutation is loading, there is a prop for that. + const { ...props } = useMutation((formData: IForm) => + queryAPI.create(formData) + ); + + return ( + // ... + ); +}; +``` + +### Page Routing + +**Deprecated:** + +Redux was used to manage redirecting to different pages of the app. + +**Current:** + +We use React Router directly to navigate between pages. For page components, +React Router (v3) supplies a `router` prop that can be easily accessed. +When needed, the `router` object contains a `push` function that redirects +a user to whatever page desired. For example: + +```typescript +// page +import PATHS from "router/paths"; + +interface IPageProps { + router: any; // no typing in react-router v3 +} + +const PageOrComponent = ({ + router, +}: IPageProps) => { + const doSomething = async () => { + router.push(PATHS.HOME); + }; + + return ( + // ... + ); +}; +``` + +### Other + +**Local states** + +Our first line of defense for state management is local states (i.e. `useState`). We +use local states to keep pages/components separate from one another and easy to +maintain. If states need to be passed to direct children, then prop-drilling should +suffice as long as we do not go more than two levels deep. Otherwise, if states need +to be used across multiple unrelated components or 3+ levels from a parent, +then the [app's context](#react-context) should be used. + +**File size** + +The recommend line limit per page/component is 500 lines. This is only a recommendation. +Larger files are to be split into multiple files if possible. \ No newline at end of file diff --git a/frontend/README_deprecated.md b/frontend/README_deprecated.md new file mode 100644 index 0000000000..4b1360d1da --- /dev/null +++ b/frontend/README_deprecated.md @@ -0,0 +1,128 @@ +**This documentation has many deprecated patterns. Please follow the current [README](./README.md).** + +# Fleet Front-End + +**Note: Redux is deprecated.** + +The Fleet front-end is a Single Page Application using React and Redux. + +## Running the Fleet web app + +For details instruction on building and serving the Fleet web application +consult the [Contributing documentation](../docs/3-Contributing/README.md). + +## Directory Structure + +Component directories in the Fleet front-end application encapsulate the entire +component, including files for the component, helper functions, styles, and tests. The +typical directory structure for a component is as follows: + +**Note: `.jsx` and `.js` is deprecated.** +``` +|-- ComponentName +| |-- _styles.scss +| |-- ComponentName.jsx +| |-- ComponentName.tests.jsx // deprecated +| |-- helpers.js +| |-- helpers.tests.js +| |-- index.js +``` + +- `_styles.scss`: The component css styles +- `ComponentName.jsx`: The React component +- `ComponentName.tests.jsx`: The React component tests `Deprecated` +- `helpers.js`: Helper functions used by the component +- `helpers.tests.js`: Tests for the component's helper functions +- `index.js`: Exports the React component + - This file is helpful as it allows other components to import the component + by it's directory name. Without this file the component name would have to + be duplicated during imports (`components/ComponentName` vs. `components/ComponentName/ComponentName`). + +### [app_constants](./app_constants) + +The app_constants directory exports the constants used in the app. Examples +include the app's URL paths, settings, and http statuses. When building features +that require constants, the constants should be added here for accessibility +throughout the application. + +### [components](./components) + +The component directory contains the React components rendered by pages. They +are typically not connected to the redux state but receive props from their +parent components to render data and handle user interactions. + +### [interfaces](./interfaces) + +**Note: PropTypes is deprecated.** + +Files in the interfaces directory are used to specify the PropTypes for a reusable Fleet +entity. This is designed to DRY up the code and increase re-usability. These +interfaces are imported into component files and implemented when defining the +component's PropTypes. + +### [fleet](./fleet) `Deprecated` + +The default export of the `fleet` directory is the API client. More info can be +found at the [API client documentation page](./fleet/README.md). + +### [layouts](https://github.com/fleetdm/fleet/tree/main/frontend/layouts) + +The Fleet application has only 1 layout, the [Core Layout](./layouts/CoreLayout/CoreLayout.jsx). +The Layout is rendered from the [router](./router/index.tsx) and are used to set up the general app UI (header, sidebar) and render child components. +The child components rendered by the layout are typically page components. + +### [pages](./pages) + +Page components are React components typically rendered from the [router](./router). +These components are connected to redux state and are used to gather data from +redux and pass that data to child components (located in the [components +directory](./components). As +connected components, Pages are also used to dispatch actions. Actions +dispatched from Pages are intended to update redux state and oftentimes include +making a call to the Fleet API. + +### [redux](./redux) `Deprecated` + +The redux directory holds all of the application's redux middleware, actions, +and reducers. The redux directory also creates the [store](./redux/store.js) which is used in the router. +More information about the redux configuration can be found at the [Redux +Documentation page](./redux/README.md) + +### [router](./router) + +The router directory is where the react router lives. The router decides which +component will render at a given URL. Components rendered from the router are +typically located in the [pages directory](./pages). The router directory also holds a `paths` +file which holds the application paths as string constants for reference +throughout the app. These paths are typically referenced from the [App +Constants](./app_constants) object. + +### [styles](./styles) + +The styles directory contains the general app style setup and variables. It +includes variables for the app color hex codes, fonts (families, weights and sizes), and padding. + +### [templates](./templates) + +The templates directory contains the HTML file that renders the React application via including the `bundle.js` +and `bundle.css` files. The HTML page also includes the HTML element in which the React application is mounted. + +### [test](./test) `Deprecated` + +The test directory includes test helpers, API request mocks, and stubbed data entities for use in test files. +More on test helpers, stubs, and request mocks [here](./test/README.md). + +### [utilities](./utilities) + +The utilities directory contains re-usable functions for use throughout the +application. The functions include helpers to convert an array of objects to +CSV, debounce functions to prevent multiple form submissions, format API errors, +etc. + +## Forms + +For details on creating a Fleet form visit the [Fleet Form Documentation](./components/forms/README.md). + +## API Client + +For details on the Fleet API Client visit the [Fleet API Client Documentation](./fleet/README.md). From a905cb3be58ad51104adeb921b7425d9fb1bb4c8 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Thu, 16 Sep 2021 12:47:04 -0700 Subject: [PATCH 11/82] Update SSO configuration docs (#2092) - Include full example for Google IDP configuration --- .../workflows/markdown-link-check-config.json | 3 + docs/2-Deploying/2-Configuration.md | 92 ++++++++++++++----- 2 files changed, 74 insertions(+), 21 deletions(-) diff --git a/.github/workflows/markdown-link-check-config.json b/.github/workflows/markdown-link-check-config.json index 79ac5c0107..d8da76dcb6 100644 --- a/.github/workflows/markdown-link-check-config.json +++ b/.github/workflows/markdown-link-check-config.json @@ -12,6 +12,9 @@ { "pattern": "fleet.corp.example.com" }, + { + "pattern": "fleet.example.com" + }, { "pattern": "/server/datastore/mysql/migrations/" }, diff --git a/docs/2-Deploying/2-Configuration.md b/docs/2-Deploying/2-Configuration.md index b7da00f7f6..89e0568667 100644 --- a/docs/2-Deploying/2-Configuration.md +++ b/docs/2-Deploying/2-Configuration.md @@ -1444,7 +1444,7 @@ sudo systemctl daemon-reload sudo systemctl restart fleet.service ``` -## Configuring Single Sign On +## Configuring Single Sign On (SSO) Fleet supports SAML single sign on capability. @@ -1454,33 +1454,22 @@ Fleet supports the SAML Web Browser SSO Profile using the HTTP Redirect Binding. ### Identity Provider (IDP) Configuration -Setting up the connected application (Fleet) with an identity provider generally requires the following information: +Setting up the service provider (Fleet) with an identity provider generally requires the following information: - _Assertion Consumer Service_ - This is the call back URL that the identity provider - will use to send security assertions to Fleet. In Okta, this field is called _Single sign on URL_. The value that you supply will be a fully qualified URL - consisting of your Fleet web address and the callback path `/api/v1/fleet/sso/callback`. For example, - if your Fleet web address is https://fleet.acme.org, then the value you would - use in the identity provider configuration would be: + will use to send security assertions to Fleet. In Okta, this field is called _Single sign on URL_. On Google it is "ACS URL". The value that you supply will be a fully qualified URL consisting of your Fleet web address and the callback path `/api/v1/fleet/sso/callback`. For example, if your Fleet web address is https://fleet.example.com, then the value you would use in the identity provider configuration would be: ``` - https://fleet.acme.org/api/v1/fleet/sso/callback + https://fleet.example.com/api/v1/fleet/sso/callback ``` -- _Entity ID_ - This value is a URI that you define. It identifies your Fleet instance as the service provider that issues authorization requests. The value must exactly match the Entity ID that you define in the Fleet SSO configuration. +- _Entity ID_ - This value is an identifier that you choose. It identifies your Fleet instance as the service provider that issues authorization requests. The value must exactly match the Entity ID that you define in the Fleet SSO configuration. - _Name ID Format_ - The value should be `urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress`. This may be shortened in the IDP setup to something like `email` or `EmailAddress`. - _Subject Type (Application username in Okta)_ - `email`. -After supplying the above information, the IDP will generate an issuer URI and a metadata URL that will be used to configure Fleet as a service provider. - - #### Example Okta IDP Configuration - - ![Example Okta IDP Configuration](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/okta-idp-setup.png) - -> The names of the items required to configure an Identity Provider may vary from provider to provider and may not conform to the SAML spec. - -> Individual users must also be setup on the IDP before they can sign in to Fleet. +After supplying the above information, the IDP will generate an issuer URI and a metadata that will be used to configure Fleet as a service provider. ### Fleet SSO Configuration @@ -1490,12 +1479,10 @@ If your IDP supports dynamic configuration, like Okta, you only need to provide Otherwise, the following values are required: -- _Identity Provider Name_ - A human friendly name of the IDP. +- _Identity Provider Name_ - A human readable name of the IDP. This is rendered on the login page. - _Entity ID_ - A URI that identifies your Fleet instance as the issuer of authorization - requests. Assuming your company name is Acme, an example might be `fleet.acme.org` although - the value could be anything as long as it is unique to Fleet as a service provider - and matches the entity provider value used in the IDP configuration. + requests (eg. `fleet.example.com`). This much match the _Entity ID_ configured with the IDP. - _Issuer URI_ - This value is obtained from the IDP. @@ -1519,6 +1506,69 @@ It is strongly recommended that at least one admin user is set up to use the tra based log in so that there is a fallback method for logging into Fleet in the event of SSO configuration problems. +#### Okta IDP Configuration + +![Example Okta IDP Configuration](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/okta-idp-setup.png) + +> The names of the items required to configure an Identity Provider may vary from provider to provider and may not conform to the SAML spec. + +> Individual users must also be setup on the IDP before they can sign in to Fleet. + +### Google Workspace IDP Configuration + +Follow these steps to configure Fleet SSO with Google Workspace. This will require administrator permissions in Google Workspace. + +1. Navigate to the [Web and Mobile Apps](https://admin.google.com/ac/apps/unified) section of the Google Workspace dashboard. Click _Add App -> Add custom SAML app_. + +Screen Shot 2021-09-15 at 2 47 36 PM + +2. Enter `Fleet` for the _App name_ and click _Continue_. + +Screen Shot 2021-09-15 at 2 50 06 PM + +3. Click _Download Metadata_, saving the metadata to your computer. Copy the _SSO URL_. Click _Continue_. + +Screen Shot 2021-09-15 at 2 52 56 PM + +4. In Fleet, navigate to the _Organization Settings_ page. Configure the _SAML Single Sign On Options_ section. + +- Check the _Enable Single Sign On_ checkbox. +- For _Identity Provider Name_ use `Google`. +- For _Entity ID_, use a unique identifier such as `fleet.example.com`. Note that Google seems to error when the provided ID includes `https://`. +- For _Issuer URI_, paste the _SSO URL_ copied from step 3. +- For _Metadata_, paste the contents of the downloaded metadata XML from step 3. +- All other fields can be left blank. + +Click _Update settings_ at the bottom of the page. + +Screen Shot 2021-09-15 at 5 32 25 PM + +5. In Google Workspace, configure the _Service provider details_. + +- For _ACS URL_, use `https:///api/v1/fleet/sso/callback` (eg. `https://fleet.example.com/api/v1/fleet/sso/callback`). +- For Entity ID, use **the same unique identifier from step 4** (eg. `fleet.example.com`). +- For _Name ID format_ choose `EMAIL`. +- For _Name ID_ choose `Basic Information > Primary email`. +- All other fields can be left blank. + +Click _Continue_ at the bottom of the page. + +Screen Shot 2021-09-15 at 5 36 56 PM + +6. Click _Finish_. + +Screen Shot 2021-09-15 at 2 57 20 PM + +7. Click the down arrow on the _User access_ section of the app details page. + +Screen Shot 2021-09-15 at 2 57 51 PM + +8. Check _ON for everyone_. Click _Save_. + +Screen Shot 2021-09-15 at 2 58 09 PM + +9. Enable SSO for a test user and try logging in. Note that Google sometimes takes a long time to propagate the SSO configuration, and it can help to try logging in to Fleet with an Incognito/Private window in the browser. + ## Feature flags Fleet features are sometimes gated behind feature flags. This will usually be due to not-yet-stable APIs, or not-fully-tested performance characteristics. From b15b41946f862d6c3b65c84e7ba9dd70c5ca07dc Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 16 Sep 2021 18:44:16 -0300 Subject: [PATCH 12/82] Add permissions/policy checklist (#2111) * Add permissions/policy checklist * Update .github/pull_request_template.md Co-authored-by: noahtalerman <47070608+noahtalerman@users.noreply.github.com> Co-authored-by: noahtalerman <47070608+noahtalerman@users.noreply.github.com> --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f38da8fefa..cc86dd9724 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,5 +4,6 @@ If some of the following don't apply, please write a short explanation why. - [ ] Changes file added (if needed) - [ ] Documented any API changes +- [ ] Documented any permissions changes - [ ] Added tests for all functionality - [ ] Manual QA for all functionality From 208c6276668551997da1c979e16e42639763dc60 Mon Sep 17 00:00:00 2001 From: eashaw Date: Thu, 16 Sep 2021 18:07:18 -0500 Subject: [PATCH 13/82] Link to documentation on github from fleetdm.com/docs (#2041) * added a pathInDocsFolder attribute to the markdown pages * updated call to action, and changed the link to the markdown file in the fleet repo * update comment * update comment and variable name --- website/scripts/build-static-content.js | 5 +++++ website/views/pages/docs/basic-documentation.ejs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/website/scripts/build-static-content.js b/website/scripts/build-static-content.js index 2d9b24a15e..fa638ff4e5 100644 --- a/website/scripts/build-static-content.js +++ b/website/scripts/build-static-content.js @@ -293,12 +293,17 @@ module.exports = { await sails.helpers.fs.write(htmlOutputPath, htmlString); } + // Determine the path of the file in the fleet repo so we can link to + // the file on github from fleetdm.com (e.g. 1-Using-Fleet/2-fleetctl-CLI.md) + let sectionRelativeRepoPath = path.relative(path.join(topLvlRepoPath, sectionRepoPath), path.resolve(pageSourcePath)); + // Append to what will become configuration for the Sails app. builtStaticContent.markdownPages.push({ url: rootRelativeUrlPath, title: pageTitle, lastModifiedAt: lastModifiedAt, htmlId: htmlId, + sectionRelativeRepoPath: sectionRelativeRepoPath, meta: _.omit(embeddedMetadata, 'title') }); } diff --git a/website/views/pages/docs/basic-documentation.ejs b/website/views/pages/docs/basic-documentation.ejs index c0b7434fa7..6db93b1b0c 100644 --- a/website/views/pages/docs/basic-documentation.ejs +++ b/website/views/pages/docs/basic-documentation.ejs @@ -205,7 +205,7 @@

    Is there something missing?

    - If you notice something we've missed or could be improved on, please follow this link and submit a pull request to the Fleet repo. + If you notice something we’ve missed, or that could be improved, please click here to edit this page.

    From 8c7155aba16812dc6a9f2ab1c63a44ea2fa15dcc Mon Sep 17 00:00:00 2001 From: Martavis Parker <47053705+martavis@users.noreply.github.com> Date: Thu, 16 Sep 2021 16:36:25 -0700 Subject: [PATCH 14/82] Flaky ManageHosts page for observers (#2116) * fixed blank hosts screen; 403 for enroll secret * lint fix * added changes log --- changes/2112-flaky-observer-hosts | 1 + frontend/components/App/App.tsx | 22 ++++++++++-- frontend/context/app.tsx | 32 ++++++++--------- .../hosts/ManageHostsPage/ManageHostsPage.jsx | 34 ++++++++++--------- 4 files changed, 55 insertions(+), 34 deletions(-) create mode 100644 changes/2112-flaky-observer-hosts diff --git a/changes/2112-flaky-observer-hosts b/changes/2112-flaky-observer-hosts new file mode 100644 index 0000000000..8017ebafb2 --- /dev/null +++ b/changes/2112-flaky-observer-hosts @@ -0,0 +1 @@ +- Fixed intermittent blank screen for observers on manage hosts page \ No newline at end of file diff --git a/frontend/components/App/App.tsx b/frontend/components/App/App.tsx index 6ba08768d8..0f5bfd600c 100644 --- a/frontend/components/App/App.tsx +++ b/frontend/components/App/App.tsx @@ -26,9 +26,15 @@ interface IRootState { const App = ({ children }: IAppProps) => { const dispatch = useDispatch(); - const { setCurrentUser, setConfig } = useContext(AppContext); const user = useSelector((state: IRootState) => state.auth.user); const queryClient = new QueryClient(); + const { + setCurrentUser, + setConfig, + currentUser, + isGlobalObserver, + isOnlyObserver, + } = useContext(AppContext); useDeepEffect(() => { // on page refresh @@ -41,10 +47,22 @@ const App = ({ children }: IAppProps) => { dispatch(getConfig()) .then((config: IConfig) => setConfig(config)) .catch(() => false); - dispatch(getEnrollSecret()).catch(() => false); } }, [user]); + useDeepEffect(() => { + const canGetEnrollSecret = + currentUser && + typeof isGlobalObserver !== "undefined" && + !isGlobalObserver && + typeof isOnlyObserver !== "undefined" && + !isOnlyObserver; + + if (canGetEnrollSecret) { + dispatch(getEnrollSecret()).catch(() => false); + } + }, [currentUser, isGlobalObserver, isOnlyObserver]); + const wrapperStyles = classnames("wrapper"); return ( diff --git a/frontend/context/app.tsx b/frontend/context/app.tsx index 488a54ab2c..31b0ee5c36 100644 --- a/frontend/context/app.tsx +++ b/frontend/context/app.tsx @@ -11,14 +11,14 @@ type Props = { type InitialStateType = { currentUser: IUser | null; config: IConfig | null; - isFreeTier: boolean; - isPremiumTier: boolean; - isGlobalAdmin: boolean; - isGlobalMaintainer: boolean; - isGlobalObserver: boolean; - isOnGlobalTeam: boolean; - isAnyTeamMaintainer: boolean; - isOnlyObserver: boolean; + isFreeTier: boolean | undefined; + isPremiumTier: boolean | undefined; + isGlobalAdmin: boolean | undefined; + isGlobalMaintainer: boolean | undefined; + isGlobalObserver: boolean | undefined; + isOnGlobalTeam: boolean | undefined; + isAnyTeamMaintainer: boolean | undefined; + isOnlyObserver: boolean | undefined; setCurrentUser: (user: IUser) => void; setConfig: (config: IConfig) => void; }; @@ -26,14 +26,14 @@ type InitialStateType = { const initialState = { currentUser: null, config: null, - isFreeTier: false, - isPremiumTier: false, - isGlobalAdmin: false, - isGlobalMaintainer: false, - isGlobalObserver: false, - isOnGlobalTeam: false, - isAnyTeamMaintainer: false, - isOnlyObserver: false, + isFreeTier: undefined, + isPremiumTier: undefined, + isGlobalAdmin: undefined, + isGlobalMaintainer: undefined, + isGlobalObserver: undefined, + isOnGlobalTeam: undefined, + isAnyTeamMaintainer: undefined, + isOnlyObserver: undefined, setCurrentUser: () => null, setConfig: () => null, }; diff --git a/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.jsx b/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.jsx index b855035180..469a867626 100644 --- a/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.jsx +++ b/frontend/pages/hosts/ManageHostsPage/ManageHostsPage.jsx @@ -223,12 +223,29 @@ export class ManageHostsPage extends PureComponent { /* eslint-enable no-alert, react/no-did-mount-set-state */ } - componentWillReceiveProps() { + /* eslint-disable react/no-did-update-set-state */ + componentDidUpdate() { + // TODO: Very temporary until this component becomes functional + // this was so we could remove redux for selectedOsqueryTable - 8/31/21 - MP + if ( + !isEqual( + this.context.selectedOsqueryTable, + this.state.selectedOsqueryTable + ) + ) { + const { selectedOsqueryTable } = this.context; + this.setState({ selectedOsqueryTable }); + } + + // end TODO + const { config, dispatch, isPremiumTier } = this.props; const { isConfigLoaded, isTeamsLoaded, isTeamsLoading } = this.state; + if (!isConfigLoaded && !isEmpty(config)) { this.setState({ isConfigLoaded: true }); } + if (isConfigLoaded && isPremiumTier && !isTeamsLoaded && !isTeamsLoading) { this.setState({ isTeamsLoading: true }); dispatch(teamActions.loadAll({})) @@ -254,21 +271,6 @@ export class ManageHostsPage extends PureComponent { }); } } - - // TODO: Very temporary until this component becomes functional - // this was so we could remove redux for selectedOsqueryTable - 8/31/21 - MP - /* eslint-disable react/no-did-update-set-state */ - componentDidUpdate() { - if ( - !isEqual( - this.context.selectedOsqueryTable, - this.state.selectedOsqueryTable - ) - ) { - const { selectedOsqueryTable } = this.context; - this.setState({ selectedOsqueryTable }); - } - } /* eslint-enable react/no-did-update-set-state */ componentWillUnmount() { From 62c93eae0d3c24f85f26670dbe3885b9baae5d4c Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Thu, 16 Sep 2021 22:33:25 -0500 Subject: [PATCH 15/82] clarify which Fleet environments receive data when usage analytics are enabled (#2106) --- website/api/controllers/webhooks/receive-usage-analytics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/api/controllers/webhooks/receive-usage-analytics.js b/website/api/controllers/webhooks/receive-usage-analytics.js index 5b3fc6bcb9..7895e1ca3e 100644 --- a/website/api/controllers/webhooks/receive-usage-analytics.js +++ b/website/api/controllers/webhooks/receive-usage-analytics.js @@ -4,7 +4,7 @@ module.exports = { friendlyName: 'Receive usage analytics', - description: '', + description: 'Receive anonymous usage analytics from deployments of Fleet running in production. (Not fleetctl preview or dev-mode deployments.)', inputs: { From d85756a1ecda99e0ecf6a0c4a9c913f1664ac928 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Thu, 16 Sep 2021 22:33:43 -0500 Subject: [PATCH 16/82] Delete why-fleet.md (#2120) will come back to this in https://github.com/orgs/fleetdm/projects/21#column-15983311 --- handbook/why-fleet.md | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 handbook/why-fleet.md diff --git a/handbook/why-fleet.md b/handbook/why-fleet.md deleted file mode 100644 index 54ad90368f..0000000000 --- a/handbook/why-fleet.md +++ /dev/null @@ -1,43 +0,0 @@ -## Why Fleet? - -We believe device management should be safe, easy, and transparent to end users. - -### The premier osquery fleet manager. -Fleet makes it easier to query and track your servers, containers, and laptops. It extends osquery to answer questions about multiple devices at the same time and provides log streams that enable automated threat detection. You can run it on your own hardware or deploy it in any cloud. - -#### Realtime visibility -- Share with any team (ops, security, helpdesk, etc) -- Unmetered access to user seats -- Intuitive user interface -- Verify remediations were actually successful -- Make compliance goals highly visible -- Track progress towards compliance goals together -- Auto-ticket any compliance regressions -- Reliable, efficient source of truth - -#### Get flexible -- Receive data from any device -- Send data anywhere it needs to go -- Express and track custom compliance goals -- Integrate existing business processes -- Play nice with internal tools and requirements -- Self-managed (cloud first, but can still be within your VPC) -- Complete control (no need to rely on 3rd party vendors, reduced risk of getting hacked) -- Deploy anywhere - -#### Developer productivity -- Programmable (REST API, ±gRPC) -- Command line tool (`fleetctl`) -- "Infrastructure as code"-ready (YAML) -- All UI functionality accessible via APIs -- Focus on value, not boilerplate -- Skip reinventing the wheel -- Ad hoc audits (live queries answer in seconds) -- 100% source code available - -#### Unlock efficiencies -- Designed for enterprise (from day 1) -- Built by osquery creators (Zach Wasserman + Mike Arpaia) -- Battle-tested in production (3+ years) -- Learn from peers' mistakes & best practices -- Widespread adoption in F1000 From 14e46cf7fa02b81c656f772f4c53355f10042488 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Fri, 17 Sep 2021 10:29:31 -0500 Subject: [PATCH 17/82] Stub growth handbook (#2096) --- handbook/growth.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 handbook/growth.md diff --git a/handbook/growth.md b/handbook/growth.md new file mode 100644 index 0000000000..694a6b5e5e --- /dev/null +++ b/handbook/growth.md @@ -0,0 +1,8 @@ +# Growth handbook + +Welcome to the Fleet growth handbook. + +TODO + + + From cf793ff3a0ed8cd74c9b31924c8c0673481e30a2 Mon Sep 17 00:00:00 2001 From: eashaw Date: Fri, 17 Sep 2021 11:14:05 -0500 Subject: [PATCH 18/82] Implement redirect in view-basic-documentation (#2083) * handling redirect * clean up redirect/notFound flow --- .../docs/view-basic-documentation.js | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/website/api/controllers/docs/view-basic-documentation.js b/website/api/controllers/docs/view-basic-documentation.js index 769474f0ad..525bee33c2 100644 --- a/website/api/controllers/docs/view-basic-documentation.js +++ b/website/api/controllers/docs/view-basic-documentation.js @@ -44,14 +44,26 @@ module.exports = { // console.log('pageUrlSuffix:',pageUrlSuffix); // console.log('SECTION_URL_PREFIX + "/" + _.trim(pageUrlSuffix, "/"):',SECTION_URL_PREFIX + '/' + _.trim(pageUrlSuffix, '/')); // console.log('thisPage:',thisPage); - if (!thisPage) { - throw 'notFound'; - } - if (false) { - // TODO: add "redirect" exit and handle mismatched capitalization / extra slashes by redirecting to the correct URL. e.g. "http://localhost:2024/docs//usiNG-fleet///" Partial example of this: https://github.com/sailshq/sailsjs.com/blob/b53c6e6a90c9afdf89e5cae00b9c9dd3f391b0e7/api/controllers/documentation/view-documentation.js#L161-L166 - let revisedUrl = 'todo'; - throw {redirect: revisedUrl}; + // Setting a flag if the pageUrlSuffix doesn't match any existing page, or if the page it matches doesn't exactly match the pageUrlSuffix provided + // Note: because this also handles the docs landing page and a pageUrlSuffix might not have provided, we set this flag to false if the url is just '/docs' + let needsRedirectMaybe = (!thisPage || (thisPage.url !== '/docs/'+pageUrlSuffix && thisPage.url !== '/docs')); + + if (needsRedirectMaybe) { + // Creating a lower case, repeating-slashless pageUrlSuffix + let multipleSlashesRegex = /\/{2,}/g; + let modifiedPageUrlSuffix = pageUrlSuffix.toLowerCase().replace(multipleSlashesRegex, '/'); + // Finding the appropriate page content using the modified pageUrlSuffix. + let revisedPage = _.find(sails.config.builtStaticContent.markdownPages, { + url: _.trimRight(SECTION_URL_PREFIX + '/' + _.trim(modifiedPageUrlSuffix, '/'), '/') + }); + if(revisedPage.url) { + // If we matched a page with the modified pageUrlSuffix, then redirect to that. + throw {redirect: revisedPage.url}; + } else { + // If no page was found, throw a 404 error. + throw 'notFound'; + } } // Respond with view. From 2c4ee75bc2054bf33d2174f02d5d847ab24283f8 Mon Sep 17 00:00:00 2001 From: AndrewB Date: Fri, 17 Sep 2021 12:19:30 -0400 Subject: [PATCH 19/82] Update 4-Committing-Changes.md (#2047) --- docs/3-Contributing/4-Committing-Changes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/3-Contributing/4-Committing-Changes.md b/docs/3-Contributing/4-Committing-Changes.md index ac99857ae9..88694c9b38 100644 --- a/docs/3-Contributing/4-Committing-Changes.md +++ b/docs/3-Contributing/4-Committing-Changes.md @@ -12,6 +12,9 @@ For significant changes, it is a good idea to discuss the proposal with the Flee Please keep in mind that any code merged to the Fleet repository becomes the responsibility of the Fleet team to maintain. Because of this, we are careful to ensure any contributions fit Fleet's vision, are well-tested, and high quality. We will work with contributors to ensure the appropriate standards are met. +## Fleet Device Management team members +Fleet Device Management team members may not copy queries from external sources except when that content has an explicit license allowing such use, or permission has been granted by the creator. + ## Pull Requests Each developer (internal or external) creates a fork of the Fleet repository, committing changes to a branch within their fork. Changes are submitted by PR to be merged into Fleet. From 1e87caa1aa60b37a4b45267eaa7589019e1c7b05 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Fri, 17 Sep 2021 10:04:51 -0700 Subject: [PATCH 20/82] Improve Cypress E2E testing stability (#2117) - Slight redesigns in some tests. - Enable default retry in run mode. - Disable flaky team flow test. --- cypress-free.json | 6 +- cypress-premium.json | 6 +- cypress.json | 6 +- cypress/integration/all/app/hosts.spec.ts | 4 +- .../integration/all/app/manageUsers.spec.ts | 10 - .../integration/all/app/resetsessions.spec.ts | 13 +- cypress/integration/premium/teamflow.spec.ts | 22 +- cypress/support/commands.ts | 4 + package.json | 2 +- yarn.lock | 518 +++++++++++++----- 10 files changed, 426 insertions(+), 165 deletions(-) diff --git a/cypress-free.json b/cypress-free.json index d2b6814b8c..91f97117f0 100644 --- a/cypress-free.json +++ b/cypress-free.json @@ -1,5 +1,9 @@ { "baseUrl": "https://localhost:8642", "fixturesFolder": false, - "testFiles": "{all,free}/**/*.spec.ts" + "testFiles": "{all,free}/**/*.spec.ts", + "retries": { + "runMode": 1, + "openMode": 0 + } } diff --git a/cypress-premium.json b/cypress-premium.json index 86c1807f11..394960d126 100644 --- a/cypress-premium.json +++ b/cypress-premium.json @@ -1,5 +1,9 @@ { "baseUrl": "https://localhost:8642", "fixturesFolder": false, - "testFiles": "{all,premium}/**/*.spec.ts" + "testFiles": "{all,premium}/**/*.spec.ts", + "retries": { + "runMode": 1, + "openMode": 0 + } } diff --git a/cypress.json b/cypress.json index 73960449f2..b15968f2da 100644 --- a/cypress.json +++ b/cypress.json @@ -2,5 +2,9 @@ "baseUrl": "https://localhost:8642", "fixturesFolder": false, "testFiles": "**/*.ts", - "defaultCommandTimeout": 8000 + "defaultCommandTimeout": 8000, + "retries": { + "runMode": 1, + "openMode": 0 + } } diff --git a/cypress/integration/all/app/hosts.spec.ts b/cypress/integration/all/app/hosts.spec.ts index 39ae559ab3..6131989072 100644 --- a/cypress/integration/all/app/hosts.spec.ts +++ b/cypress/integration/all/app/hosts.spec.ts @@ -10,6 +10,7 @@ describe( cy.setup(); cy.login(); cy.addDockerHost(); + cy.clearDownloads(); }); afterEach(() => { @@ -45,9 +46,6 @@ describe( cy.get("input[disabled]").should("have.value", contents); }); - // ensure load - cy.wait(5000); // eslint-disable-line cypress/no-unnecessary-waiting - // Wait until the host becomes available (usually immediate in local // testing, but may vary by environment). cy.waitUntil( diff --git a/cypress/integration/all/app/manageUsers.spec.ts b/cypress/integration/all/app/manageUsers.spec.ts index 674ee386fa..5c6f25dc71 100644 --- a/cypress/integration/all/app/manageUsers.spec.ts +++ b/cypress/integration/all/app/manageUsers.spec.ts @@ -6,15 +6,7 @@ describe("Manage Users", () => { }); it("Searching for a user", () => { - cy.intercept({ - method: "GET", - url: "/api/v1/fleet/users", - }).as("getUsers"); - cy.visit("/settings/users"); - cy.url().should("match", /\/settings\/users$/i); - - cy.wait("@getUsers"); cy.findByText("admin@example.com").should("exist"); cy.findByText("maintainer@example.com").should("exist"); @@ -23,8 +15,6 @@ describe("Manage Users", () => { cy.findByPlaceholderText("Search").type("admin"); - cy.wait("@getUsers"); - cy.findByText("admin@example.com").should("exist"); cy.findByText("maintainer@example.com").should("not.exist"); cy.findByText("observer@example.com").should("not.exist"); diff --git a/cypress/integration/all/app/resetsessions.spec.ts b/cypress/integration/all/app/resetsessions.spec.ts index c9bba80710..0a9e1e9100 100644 --- a/cypress/integration/all/app/resetsessions.spec.ts +++ b/cypress/integration/all/app/resetsessions.spec.ts @@ -20,14 +20,11 @@ describe("Reset user sessions flow", () => { // first select the table cell with the user's email address then go back up to the containing row // so we can select reset sessions from actions dropdown - cy.get("tbody>tr>td") - .contains(/admin@example.com/i) - .parent() - .parent() - .within(() => { - cy.findByText(/actions/i).click(); - cy.findByText(/reset sessions/i).click(); - }); + cy.contains("div.Select-placeholder", /actions/i); + // eslint-disable-next-line cypress/no-unnecessary-waiting + cy.wait(500); + cy.contains("div.Select-placeholder", /actions/i).click(); + cy.contains(/reset sessions/i).click(); cy.get(".modal__modal_container").within(() => { cy.findByText(/reset sessions/i).should("exist"); diff --git a/cypress/integration/premium/teamflow.spec.ts b/cypress/integration/premium/teamflow.spec.ts index 2d26bcfdcb..0b9e736e68 100644 --- a/cypress/integration/premium/teamflow.spec.ts +++ b/cypress/integration/premium/teamflow.spec.ts @@ -5,12 +5,13 @@ describe("Teams flow", () => { cy.viewport(1200, 660); }); + /* TODO fix and reenable + This test is causing major flake issues due to the dropdown menu + it("Create, edit, and delete a team successfully", () => { cy.visit("/settings/teams"); - cy.wait(2000); // eslint-disable-line cypress/no-unnecessary-waiting - - cy.findByRole("button", { name: /create team/i }).click(); + cy.findByRole("button", { name: /create team/i }).click({ force: true }); cy.findByLabelText(/team name/i) .click() @@ -28,16 +29,20 @@ describe("Teams flow", () => { cy.findByText(/agent options/i).click(); - cy.get(".ace_content") - .click() - .type("{selectall}{backspace}apiVersion: v1{enter}kind: options"); + cy.contains(".ace_content", "config:"); + cy.get(".ace_text-input") + .first() + .focus() + .type("{selectall}{backspace}config:\n options:"); cy.findByRole("button", { name: /save options/i }).click(); + cy.contains("span", /successfully saved/i); + cy.visit("/settings/teams/1/options"); - cy.contains(/apiVersion: v1/i).should("be.visible"); - cy.contains(/kind: options/i).should("be.visible"); + cy.contains(/config:/i).should("be.visible"); + cy.contains(/options:/i).should("be.visible"); cy.visit("/settings/teams"); @@ -72,4 +77,5 @@ describe("Teams flow", () => { cy.findByText(/mystic/i).should("not.exist"); }); + */ }); diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index 9af750deac..b73c3ccae9 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -285,3 +285,7 @@ Cypress.Commands.add("stopDockerHost", () => { }, }); }); + +Cypress.Commands.add("clearDownloads", () => { + cy.exec(`rm -rf ${Cypress.config("downloadsFolder")}`); +}); diff --git a/package.json b/package.json index cfa86f317f..340b3a1b8b 100644 --- a/package.json +++ b/package.json @@ -176,7 +176,7 @@ "babel-jest": "^23.4.2", "babel-loader": "^8.2.1", "css-loader": "0.28.11", - "cypress": "^6.9.1", + "cypress": "8.4.0", "cypress-wait-until": "^1.7.1", "enzyme": "^3.11.0", "enzyme-adapter-react-16": "^1.15.6", diff --git a/yarn.lock b/yarn.lock index ae51c26324..4afebf1bc4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1122,10 +1122,10 @@ date-fns "^1.27.2" figures "^1.7.0" -"@cypress/request@^2.88.5": - version "2.88.5" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.5.tgz#8d7ecd17b53a849cfd5ab06d5abe7d84976375d7" - integrity sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA== +"@cypress/request@^2.88.5", "@cypress/request@^2.88.6": + version "2.88.6" + resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.6.tgz#a970dd675befc6bdf8a8921576c01f51cc5798e9" + integrity sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ== dependencies: aws-sign2 "~0.7.0" aws4 "^1.8.0" @@ -1140,13 +1140,12 @@ isstream "~0.1.2" json-stringify-safe "~5.0.1" mime-types "~2.1.19" - oauth-sign "~0.9.0" performance-now "^2.1.0" qs "~6.5.2" safe-buffer "^5.1.2" tough-cookie "~2.5.0" tunnel-agent "^0.6.0" - uuid "^3.3.2" + uuid "^8.3.2" "@cypress/xvfb@^1.2.4": version "1.2.4" @@ -1692,6 +1691,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.50.tgz#e9b2e85fafc15f2a8aa8fdd41091b983da5fd6ee" integrity sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w== +"@types/node@^14.14.31": + version "14.17.16" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.17.16.tgz#2b9252bd4fdf0393696190cd9550901dd967c777" + integrity sha512-WiFf2izl01P1CpeY8WqFAeKWwByMueBEkND38EcN8N68qb0aDG3oIS1P5MhAX5kUdr469qRyqsY/MjanLjsFbQ== + "@types/node@^14.14.37": version "14.14.37" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.37.tgz#a3dd8da4eb84a996c36e331df98d82abd76b516e" @@ -1784,15 +1788,15 @@ "@types/prop-types" "*" csstype "^3.0.2" -"@types/sinonjs__fake-timers@^6.0.1": - version "6.0.2" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.2.tgz#3a84cf5ec3249439015e14049bd3161419bf9eae" - integrity sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg== +"@types/sinonjs__fake-timers@^6.0.1", "@types/sinonjs__fake-timers@^6.0.2": + version "6.0.3" + resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08" + integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g== "@types/sizzle@^2.3.2": - version "2.3.2" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.2.tgz#a811b8c18e2babab7d542b3365887ae2e4d9de47" - integrity sha512-7EJYyKTL7tFR8+gDbB6Wwz/arpGa0Mywk1TJbNzKzHtzbwVmY4HR9WqS5VV7dsBUKQmPNr192jHr/VpBluj/hg== + version "2.3.3" + resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" + integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== "@types/sockjs-client@^1.5.1": version "1.5.1" @@ -1828,6 +1832,13 @@ dependencies: "@types/yargs-parser" "*" +"@types/yauzl@^2.9.1": + version "2.9.2" + resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" + integrity sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA== + dependencies: + "@types/node" "*" + "@typescript-eslint/eslint-plugin@^4.15.2": version "4.15.2" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.15.2.tgz#981b26b4076c62a5a55873fbef3fe98f83360c61" @@ -2110,6 +2121,14 @@ acorn@^8.1.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.1.1.tgz#fb0026885b9ac9f48bac1e185e4af472971149ff" integrity sha512-xYiIVjNuqtKXMxlRMDc6mZUhXehod4a3gbZ1qRlM7icK4EbxUFNLhWoPblCvFtB2Y9CIqHP3CF/rdxLItaQv8g== +aggregate-error@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" + integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== + dependencies: + clean-stack "^2.0.0" + indent-string "^4.0.0" + airbnb-prop-types@^2.16.0: version "2.16.0" resolved "https://registry.yarnpkg.com/airbnb-prop-types/-/airbnb-prop-types-2.16.0.tgz#b96274cefa1abb14f623f804173ee97c13971dc2" @@ -2202,6 +2221,13 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.11.0" +ansi-escapes@^4.3.0: + version "4.3.2" + resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" + integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== + dependencies: + type-fest "^0.21.3" + ansi-html@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" @@ -2223,9 +2249,9 @@ ansi-regex@^4.1.0: integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^2.2.1: version "2.2.1" @@ -2277,7 +2303,7 @@ aproba@^1.0.3, aproba@^1.1.1: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== -arch@^2.1.2: +arch@^2.1.2, arch@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== @@ -2487,9 +2513,9 @@ async@^2.5.0: lodash "^4.17.11" async@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" - integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== + version "3.2.1" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.1.tgz#d3274ec66d107a47476a4c49136aacdb00665fc8" + integrity sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg== asynckit@^0.4.0: version "0.4.0" @@ -2537,9 +2563,9 @@ aws-sign2@~0.7.0: integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= aws4@^1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" - integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== axios@^0.21.1: version "0.21.1" @@ -3391,9 +3417,9 @@ balanced-match@^0.4.2: integrity sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg= balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== base64-js@^1.0.2: version "1.3.0" @@ -3445,7 +3471,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -blob-util@2.0.2: +blob-util@2.0.2, blob-util@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== @@ -3667,9 +3693,9 @@ buffer-crc32@~0.2.3: integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + version "1.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== buffer-xor@^1.0.3: version "1.0.3" @@ -3874,9 +3900,9 @@ chalk@^3.0.0: supports-color "^7.1.0" chalk@^4.0.0, chalk@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" - integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== dependencies: ansi-styles "^4.1.0" supports-color "^7.1.0" @@ -3965,6 +3991,11 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== +ci-info@^3.1.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.2.0.tgz#2876cb948a498797b5236f0095bc057d0dca38b6" + integrity sha512-dVqRX7fLUm8J6FgHJ418XuIgDLZDkYcDFTeL6TA2gt5WlIZUQrrH6EZrNClwT/H0FateUsZkGIOPRrLbP+PR9A== + cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" @@ -4012,6 +4043,11 @@ clean-css@4.2.x: dependencies: source-map "~0.6.0" +clean-stack@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" + integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== + cli-cursor@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987" @@ -4026,6 +4062,13 @@ cli-cursor@^2.0.0, cli-cursor@^2.1.0: dependencies: restore-cursor "^2.0.0" +cli-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" + integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== + dependencies: + restore-cursor "^3.1.0" + cli-table3@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.0.tgz#b7b1bc65ca8e7b5cef9124e13dc2b21e2ce4faee" @@ -4044,6 +4087,14 @@ cli-truncate@^0.2.1: slice-ansi "0.0.4" string-width "^1.0.1" +cli-truncate@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" + integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== + dependencies: + slice-ansi "^3.0.0" + string-width "^4.2.0" + cliui@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" @@ -4166,6 +4217,11 @@ colorette@^1.2.1: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== +colorette@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" + integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== + colormin@^1.0.5: version "1.1.2" resolved "https://registry.yarnpkg.com/colormin/-/colormin-1.1.2.tgz#ea2f7420a72b96881a38aae59ec124a6f7298133" @@ -4176,9 +4232,9 @@ colormin@^1.0.5: has "^1.0.1" colors@^1.1.2: - version "1.3.3" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.3.tgz#39e005d546afe01e01f9c4ca8fa50f686a01205d" - integrity sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg== + version "1.4.0" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== colors@~1.1.2: version "1.1.2" @@ -4353,11 +4409,16 @@ core-js@^2.4.0, core-js@^2.5.0: resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.9.tgz#6b4b214620c834152e179323727fc19741b084f2" integrity sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A== -core-util-is@1.0.2, core-util-is@~1.0.0: +core-util-is@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== + cosmiconfig@^5.0.0: version "5.2.1" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a" @@ -4676,47 +4737,48 @@ cypress@*: url "^0.11.0" yauzl "^2.10.0" -cypress@^6.9.1: - version "6.9.1" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-6.9.1.tgz#ce1106bfdc47f8d76381dba63f943447883f864c" - integrity sha512-/RVx6sOhsyTR9sd9v0BHI4tnDZAhsH9rNat7CIKCUEr5VPWxyfGH0EzK4IHhAqAH8vjFcD4U14tPiJXshoUrmQ== +cypress@8.4.0: + version "8.4.0" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-8.4.0.tgz#09ec06a73f1cb10121c103cba15076e659e24876" + integrity sha512-RtVgGFR06ikyMaq/VqapeqOjGaIA42PpK7F0qe1MCiFArfUuJECsLmeYaOA+1TlmNUgJNMSF5fWKkZIJr5Uc7w== dependencies: - "@cypress/listr-verbose-renderer" "^0.4.1" - "@cypress/request" "^2.88.5" + "@cypress/request" "^2.88.6" "@cypress/xvfb" "^1.2.4" - "@types/node" "12.12.50" - "@types/sinonjs__fake-timers" "^6.0.1" + "@types/node" "^14.14.31" + "@types/sinonjs__fake-timers" "^6.0.2" "@types/sizzle" "^2.3.2" - arch "^2.1.2" - blob-util "2.0.2" + arch "^2.2.0" + blob-util "^2.0.2" bluebird "^3.7.2" cachedir "^2.3.0" chalk "^4.1.0" check-more-types "^2.24.0" + cli-cursor "^3.1.0" cli-table3 "~0.6.0" commander "^5.1.0" common-tags "^1.8.0" - dayjs "^1.9.3" - debug "4.3.2" - eventemitter2 "^6.4.2" - execa "^4.0.2" + dayjs "^1.10.4" + debug "^4.3.2" + enquirer "^2.3.6" + eventemitter2 "^6.4.3" + execa "4.1.0" executable "^4.1.1" - extract-zip "^1.7.0" - fs-extra "^9.0.1" + extract-zip "2.0.1" + figures "^3.2.0" + fs-extra "^9.1.0" getos "^3.2.1" - is-ci "^2.0.0" - is-installed-globally "^0.3.2" + is-ci "^3.0.0" + is-installed-globally "~0.4.0" lazy-ass "^1.6.0" - listr "^0.14.3" - lodash "^4.17.19" + listr2 "^3.8.3" + lodash "^4.17.21" log-symbols "^4.0.0" minimist "^1.2.5" - moment "^2.29.1" ospath "^1.2.2" - pretty-bytes "^5.4.1" + pretty-bytes "^5.6.0" ramda "~0.27.1" request-progress "^3.0.0" - supports-color "^7.2.0" + supports-color "^8.1.1" tmp "~0.2.1" untildify "^4.0.0" url "^0.11.0" @@ -4753,10 +4815,10 @@ date-now@^0.1.4: resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" integrity sha1-6vQ5/U1ISK105cx9vvIAZyueNFs= -dayjs@^1.9.3: - version "1.10.4" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.4.tgz#8e544a9b8683f61783f570980a8a80eaf54ab1e2" - integrity sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw== +dayjs@^1.10.4, dayjs@^1.9.3: + version "1.10.7" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.7.tgz#2cf5f91add28116748440866a0a1d26f3a6ce468" + integrity sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig== debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: version "2.6.9" @@ -4765,7 +4827,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: dependencies: ms "2.0.0" -debug@4.3.2: +debug@4.3.2, debug@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== @@ -5176,13 +5238,20 @@ encoding@^0.1.11: dependencies: iconv-lite "^0.6.2" -end-of-stream@^1.0.0, end-of-stream@^1.1.0: +end-of-stream@^1.0.0: version "1.4.1" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" integrity sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q== dependencies: once "^1.4.0" +end-of-stream@^1.1.0: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + enhanced-resolve@4.1.0, enhanced-resolve@^4.0.0, enhanced-resolve@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz#41c7e0bfdfe74ac1ffe1e57ad6a5c6c9f3742a7f" @@ -5201,7 +5270,7 @@ enhanced-resolve@~0.9.0: memory-fs "^0.2.0" tapable "^0.1.8" -enquirer@^2.3.5: +enquirer@^2.3.5, enquirer@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== @@ -5717,7 +5786,7 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= -eventemitter2@^6.4.2: +eventemitter2@^6.4.2, eventemitter2@^6.4.3: version "6.4.4" resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.4.tgz#aa96e8275c4dbeb017a5d0e03780c65612a1202b" integrity sha512-HLU3NDY6wARrLCEwyGKRBvuWYyvW6mHYv72SJJAH3iJN3a6eVUvkjFkcxah1bcTgGVBBrFdIopBJPhCQFMLyXw== @@ -5747,20 +5816,7 @@ exec-sh@^0.3.2: resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" integrity sha512-sEFIkc61v75sWeOe72qyrqg2Qg0OuLESziUDk/O/z2qgS15y2gWVFrI6f2Qn/qw/0/NCfCEsmNA4zOjkwEZT1A== -execa@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" - integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== - dependencies: - cross-spawn "^6.0.0" - get-stream "^4.0.0" - is-stream "^1.1.0" - npm-run-path "^2.0.0" - p-finally "^1.0.0" - signal-exit "^3.0.0" - strip-eof "^1.0.0" - -execa@^4.0.0, execa@^4.0.2: +execa@4.1.0, execa@^4.0.0, execa@^4.0.2: version "4.1.0" resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== @@ -5775,6 +5831,19 @@ execa@^4.0.0, execa@^4.0.2: signal-exit "^3.0.2" strip-final-newline "^2.0.0" +execa@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" + integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== + dependencies: + cross-spawn "^6.0.0" + get-stream "^4.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + executable@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" @@ -5933,6 +6002,17 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" +extract-zip@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + extract-zip@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" @@ -5981,9 +6061,9 @@ fast-glob@^3.1.1: picomatch "^2.2.1" fast-json-stable-stringify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" - integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I= + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.4: version "2.0.6" @@ -6056,6 +6136,13 @@ figures@^2.0.0: dependencies: escape-string-regexp "^1.0.5" +figures@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" + integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== + dependencies: + escape-string-regexp "^1.0.5" + file-entry-cache@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -6287,7 +6374,7 @@ from2@^2.1.0: inherits "^2.0.1" readable-stream "^2.0.0" -fs-extra@^9.0.1: +fs-extra@^9.0.1, fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== @@ -6439,7 +6526,7 @@ get-stream@^4.0.0: dependencies: pump "^3.0.0" -get-stream@^5.0.0: +get-stream@^5.0.0, get-stream@^5.1.0: version "5.2.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== @@ -6506,7 +6593,7 @@ glob@^5.0.13: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@~7.1.1: +glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.4, glob@~7.1.1: version "7.1.4" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255" integrity sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A== @@ -6518,6 +6605,18 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, gl once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.1.3: + version "7.1.7" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" + integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + global-dirs@^2.0.1: version "2.1.0" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.1.0.tgz#e9046a49c806ff04d6c1825e196c8f0091e8df4d" @@ -6525,6 +6624,13 @@ global-dirs@^2.0.1: dependencies: ini "1.3.7" +global-dirs@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" + integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== + dependencies: + ini "2.0.0" + global-modules@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -6599,15 +6705,15 @@ globule@^1.0.0: lodash "~4.17.10" minimatch "~3.0.2" -graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6: +graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2: version "4.2.0" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b" integrity sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg== -graceful-fs@^4.2.0: - version "4.2.6" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" - integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== +graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.8" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" + integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== graceful-fs@^4.2.4: version "4.2.4" @@ -7150,6 +7256,11 @@ ini@1.3.7: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.7.tgz#a09363e1911972ea16d7a8851005d84cf09a9a84" integrity sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ== +ini@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" @@ -7285,6 +7396,13 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" +is-ci@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.0.tgz#c7e7be3c9d8eef7d0fa144390bd1e4b88dc4c994" + integrity sha512-kDXyttuLeslKAHYL/K28F2YkM3x5jvFPEw3yXbRptXydjD9rpLEz+C5K5iutY9ZiUu6AP41JdvRQwF4Iqs4ZCQ== + dependencies: + ci-info "^3.1.1" + is-core-module@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946" @@ -7453,6 +7571,14 @@ is-installed-globally@^0.3.2: global-dirs "^2.0.1" is-path-inside "^3.0.1" +is-installed-globally@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" + integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== + dependencies: + global-dirs "^3.0.0" + is-path-inside "^3.0.2" + is-negative-zero@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" @@ -7499,10 +7625,10 @@ is-observable@^1.1.0: dependencies: symbol-observable "^1.1.0" -is-path-inside@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017" - integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg== +is-path-inside@^3.0.1, is-path-inside@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== is-plain-obj@^1.0.0: version "1.1.0" @@ -7564,9 +7690,9 @@ is-stream@^1.0.1, is-stream@^1.1.0: integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= is-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" - integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-string@^1.0.4: version "1.0.4" @@ -7609,6 +7735,11 @@ is-typedarray@^1.0.0, is-typedarray@~1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= +is-unicode-supported@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" + integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== + is-utf8@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" @@ -8477,6 +8608,19 @@ listr-verbose-renderer@^0.5.0: date-fns "^1.27.2" figures "^2.0.0" +listr2@^3.8.3: + version "3.12.1" + resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.12.1.tgz#75e515b86c66b60baf253542cc0dced6b60fedaf" + integrity sha512-oB1DlXlCzGPbvWhqYBZUQEPJKqsmebQWofXG6Mpbe3uIvoNl8mctBEojyF13ZyqwQ91clCWXpwsWp+t98K4FOQ== + dependencies: + cli-truncate "^2.1.0" + colorette "^1.4.0" + log-update "^4.0.0" + p-map "^4.0.0" + rxjs "^6.6.7" + through "^2.3.8" + wrap-ansi "^7.0.0" + listr@^0.14.3: version "0.14.3" resolved "https://registry.yarnpkg.com/listr/-/listr-0.14.3.tgz#2fea909604e434be464c50bddba0d496928fa586" @@ -8639,7 +8783,7 @@ lodash@^4.0.0, lodash@^4.17.11, lodash@^4.17.2, lodash@^4.17.3, lodash@^4.17.4, resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba" integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw== -lodash@^4.15.0, lodash@^4.17.20, lodash@^4.7.0: +lodash@^4.15.0, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -8649,11 +8793,6 @@ lodash@^4.17.15: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== -lodash@^4.17.19: - version "4.17.20" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" - integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== - log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -8662,11 +8801,12 @@ log-symbols@^1.0.2: chalk "^1.0.0" log-symbols@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.0.0.tgz#69b3cc46d20f448eccdb75ea1fa733d9e821c920" - integrity sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA== + version "4.1.0" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" + integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== dependencies: - chalk "^4.0.0" + chalk "^4.1.0" + is-unicode-supported "^0.1.0" log-update@^2.3.0: version "2.3.0" @@ -8677,6 +8817,16 @@ log-update@^2.3.0: cli-cursor "^2.0.0" wrap-ansi "^3.0.1" +log-update@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" + integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== + dependencies: + ansi-escapes "^4.3.0" + cli-cursor "^3.1.0" + slice-ansi "^4.0.0" + wrap-ansi "^6.2.0" + lolex@^5.0.0: version "5.1.2" resolved "https://registry.yarnpkg.com/lolex/-/lolex-5.1.2.tgz#953694d098ce7c07bc5ed6d0e42bc6c0c6d5a367" @@ -8958,7 +9108,19 @@ mime-db@1.40.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== -mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24: +mime-db@1.49.0: + version "1.49.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed" + integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.32" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.32.tgz#1d00e89e7de7fe02008db61001d9e02852670fd5" + integrity sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A== + dependencies: + mime-db "1.49.0" + +mime-types@~2.1.24: version "2.1.24" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" integrity sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ== @@ -9125,11 +9287,16 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@2.1.2, ms@^2.1.1: +ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + nan@^2.12.1, nan@^2.13.2: version "2.14.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" @@ -9827,6 +9994,13 @@ p-map@^2.0.0: resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== +p-map@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" + integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== + dependencies: + aggregate-error "^3.0.0" + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -10463,7 +10637,7 @@ prettier@2.2.1: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== -pretty-bytes@^5.4.1: +pretty-bytes@^5.4.1, pretty-bytes@^5.6.0: version "5.6.0" resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== @@ -10572,12 +10746,12 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= -psl@^1.1.24, psl@^1.1.28: +psl@^1.1.24: version "1.2.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.2.0.tgz#df12b5b1b3a30f51c329eacbdef98f3a6e136dc6" integrity sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== @@ -11001,7 +11175,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== @@ -11014,6 +11188,19 @@ read-pkg@^5.2.0: string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@^2.2.2: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readable-stream@^3.1.1: version "3.4.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc" @@ -11457,6 +11644,14 @@ restore-cursor@^2.0.0: onetime "^2.0.0" signal-exit "^3.0.2" +restore-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" + integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== + dependencies: + onetime "^5.1.0" + signal-exit "^3.0.2" + ret@~0.1.10: version "0.1.15" resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" @@ -11523,10 +11718,10 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs@^6.3.3: - version "6.6.6" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.6.tgz#14d8417aa5a07c5e633995b525e1e3c0dec03b70" - integrity sha512-/oTwee4N4iWzAMAL9xdGKjkEHmIwupR3oXbQjCKywF1BeFohswF3vZdogbmEF6pZkOsXTzWkrZszrWpQTByYVg== +rxjs@^6.3.3, rxjs@^6.6.7: + version "6.6.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" + integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== dependencies: tslib "^1.9.0" @@ -11535,11 +11730,16 @@ safe-buffer@5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" integrity sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg== -safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2: +safe-buffer@>=5.1.0, safe-buffer@^5.1.0, safe-buffer@^5.1.1: version "5.2.0" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== +safe-buffer@^5.0.1, safe-buffer@^5.1.2: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -11821,9 +12021,9 @@ signal-exit@^3.0.0: integrity sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0= signal-exit@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" - integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== + version "3.0.4" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.4.tgz#366a4684d175b9cab2081e3681fda3747b6c51d7" + integrity sha512-rqYhcAnZ6d/vTPGghdrw7iumdcbXpsk1b8IG/rz+VWV51DM0p7XCtMoJ3qhPLIbp3tvyt3pKRbaaEMZYpHto8Q== sisteransi@^1.0.5: version "1.0.5" @@ -11850,6 +12050,15 @@ slice-ansi@0.0.4: resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35" integrity sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU= +slice-ansi@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" + integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== + dependencies: + ansi-styles "^4.0.0" + astral-regex "^2.0.0" + is-fullwidth-code-point "^3.0.0" + slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -12197,7 +12406,7 @@ string-width@^3.0.0, string-width@^3.1.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" -string-width@^4.1.0, string-width@^4.2.0: +string-width@^4.1.0: version "4.2.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== @@ -12206,6 +12415,15 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" +string-width@^4.2.0: + version "4.2.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" + integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.0" + string.prototype.matchall@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz#48bb510326fb9fdeb6a33ceaa81a6ea04ef7648e" @@ -12393,6 +12611,13 @@ supports-color@^7.0.0, supports-color@^7.1.0, supports-color@^7.2.0: dependencies: has-flag "^4.0.0" +supports-color@^8.1.1: + version "8.1.1" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + supports-hyperlinks@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz#f663df252af5f37c5d49bbd7eeefa9e0b9e59e47" @@ -12542,6 +12767,11 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" +through@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" + integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= + timers-browserify@^2.0.4: version "2.0.10" resolved "https://registry.yarnpkg.com/timers-browserify/-/timers-browserify-2.0.10.tgz#1d28e3d2aadf1d5a5996c4e9f95601cd053480ae" @@ -12737,11 +12967,16 @@ tslib@1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8" integrity sha512-f/qGG2tUkrISBlQZEjEqoZ3B2+npJjIf04H1wuAv9iA8i04Icp+61KRXxFdha22670NJopsZCIjhC3SnjPRKrQ== -tslib@^1.8.0, tslib@^1.8.1, tslib@^1.9.0: +tslib@^1.8.0, tslib@^1.8.1: version "1.10.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ== +tslib@^1.9.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" + integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== + tslint-eslint-rules@^5.4.0: version "5.4.0" resolved "https://registry.yarnpkg.com/tslint-eslint-rules/-/tslint-eslint-rules-5.4.0.tgz#e488cc9181bf193fe5cd7bfca213a7695f1737b5" @@ -12838,6 +13073,11 @@ type-fest@^0.11.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== +type-fest@^0.21.3: + version "0.21.3" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" + integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== + type-fest@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" @@ -13005,9 +13245,9 @@ upper-case@^1.1.1: integrity sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg= uri-js@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" - integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ== + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" @@ -13089,9 +13329,9 @@ utils-merge@1.0.1: integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= uuid@^3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== uuid@^7.0.3: version "7.0.3" @@ -13103,6 +13343,11 @@ uuid@^8.3.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.1.tgz#2ba2e6ca000da60fce5a196954ab241131e05a31" integrity sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg== +uuid@^8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + v8-compile-cache@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe" @@ -13470,6 +13715,15 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From bdae8d04a257fe98e9eb1c216d8e13e62b7156e1 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Sat, 18 Sep 2021 11:33:36 -0300 Subject: [PATCH 21/82] Skip saving host users and inventory if disabled (#2127) --- changes/skip-save-users-if-disabled | 1 + server/datastore/mysql/hosts.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 changes/skip-save-users-if-disabled diff --git a/changes/skip-save-users-if-disabled b/changes/skip-save-users-if-disabled new file mode 100644 index 0000000000..9c3f084555 --- /dev/null +++ b/changes/skip-save-users-if-disabled @@ -0,0 +1 @@ +* Completely skip trying to save host users and software inventory if it's disabled. diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index f54d922da6..e23566b0d6 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -146,7 +146,12 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { } } - if host.HostSoftware.Modified { + ac, err := d.AppConfig(ctx) + if err != nil { + return errors.Wrap(err, "failed to get app config to see if we need to update host users and inventory") + } + + if host.HostSoftware.Modified && ac.HostSettings.EnableSoftwareInventory { if err := saveHostSoftwareDB(ctx, tx, host); err != nil { return errors.Wrap(err, "failed to save host software") } @@ -157,8 +162,10 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { return errors.Wrap(err, "failed to save host additional") } - if err := saveHostUsersDB(ctx, tx, host); err != nil { - return errors.Wrap(err, "failed to save host users") + if ac.HostSettings.EnableHostUsers { + if err := saveHostUsersDB(ctx, tx, host); err != nil { + return errors.Wrap(err, "failed to save host users") + } } } From baa42d367e86c26cccf3a9610c7e3dc829b97881 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 20 Sep 2021 11:00:57 -0300 Subject: [PATCH 22/82] Add team policies (#2103) * Add team policies * Add team policy documentation * Add changes file * Update titles * Fix lint * Rewrite TeamAuthorize for more clarify * Explicitly use two slices for clarity * Simplify switch --- changes/issue-1893-team-policies | 1 + docs/1-Using-Fleet/3-REST-API.md | 156 ++++++++++++++++++ server/authz/authz.go | 31 ++++ server/authz/policy.rego | 7 + .../20210915144307_AddPoliciesTeamColumn.go | 25 +++ server/datastore/mysql/policies.go | 107 ++++++++++-- server/datastore/mysql/policies_test.go | 129 ++++++++++++++- server/datastore/mysql/schema.sql | 9 +- server/fleet/datastore.go | 11 +- server/fleet/global_policies.go | 1 + server/fleet/service.go | 12 ++ server/mock/datastore_mock.go | 40 +++++ server/service/handler.go | 5 + server/service/integration_enterprise_test.go | 68 +++++++- server/service/team_policies.go | 155 +++++++++++++++++ server/service/team_schedule.go | 13 +- server/service/testing_client.go | 11 +- 17 files changed, 754 insertions(+), 27 deletions(-) create mode 100644 changes/issue-1893-team-policies create mode 100644 server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go create mode 100644 server/service/team_policies.go diff --git a/changes/issue-1893-team-policies b/changes/issue-1893-team-policies new file mode 100644 index 0000000000..62e8b159ad --- /dev/null +++ b/changes/issue-1893-team-policies @@ -0,0 +1 @@ +* Add team policies. diff --git a/docs/1-Using-Fleet/3-REST-API.md b/docs/1-Using-Fleet/3-REST-API.md index d0a59d4478..f08fa28b9d 100644 --- a/docs/1-Using-Fleet/3-REST-API.md +++ b/docs/1-Using-Fleet/3-REST-API.md @@ -10,6 +10,7 @@ - [Schedule](#schedule) - [Packs](#packs) - [Policies](#policies) +- [Team Policies](#team-policies) - [Activities](#activities) - [Targets](#targets) - [Fleet configuration](#fleet-configuration) @@ -4439,6 +4440,161 @@ Hosts that do not return results for a policy's query are "Failing." --- +## Team Policies + +- [List team policies](#list-team-policies) +- [Get team policy by ID](#get-team-policy-by-id) +- [Add team policy](#add-team-policy) +- [Remove team policies](#remove-team-policies) + +_Available in Fleet Premium_ + +Team policies work the same as policies, but at the team level. + +### List team policies + +`GET /api/v1/fleet/team/{team_id}/policies` + +#### Parameters + +| Name | Type | In | Description | +| ------------------ | ------- | ---- | ------------------------------------------------------------------------------------------------------------- | +| team_id | integer | url | Defines what team id to operate on | + +#### Example + +`GET /api/v1/fleet/team/1/policies` + +##### Default response + +`Status: 200` + +``` +{ + "policies": [ + { + "id": 1, + "query_id": 2, + "query_name": "Gatekeeper enabled", + "passing_host_count": 2000, + "failing_host_count": 300, + }, + { + "id": 2, + "query_id": 3, + "query_name": "Primary disk encrypted", + "passing_host_count": 2300, + "failing_host_count": 0, + } + ] +} +``` + +### Get team policy by ID + +`GET /api/v1/fleet/team/{team_id}/policies/{id}` + +#### Parameters + +| Name | Type | In | Description | +| ------------------ | ------- | ---- | ------------------------------------------------------------------------------------------------------------- | +| team_id | integer | url | Defines what team id to operate on | +| id | integer | path | **Required.** The policy's ID. | + +#### Example + +`GET /api/v1/fleet/team/1/policies/1` + +##### Default response + +`Status: 200` + +``` +{ + "policy": { + "id": 1, + "query_id": 2, + "query_name": "Gatekeeper enabled", + "passing_host_count": 2000, + "failing_host_count": 300, + } +} +``` + +### Add team policy + +`POST /api/v1/fleet/team/{team_id}/policies` + +#### Parameters + +| Name | Type | In | Description | +| -------- | ------- | ---- | ----------------------------------- | +| team_id | integer | url | Defines what team id to operate on | +| query_id | integer | body | **Required.** The query's ID. | + +#### Example + +`POST /api/v1/fleet/team/1/policies` + +#### Request body + +``` +{ + "query_id": 12 +} +``` + +##### Default response + +`Status: 200` + +``` +{ + "policy": { + "id": 2, + "query_id": 2, + "query_name": "Primary disk encrypted", + "passing_host_count": 0, + "failing_host_count": 0, + }, +} +``` + +### Remove team policies + +`POST /api/v1/fleet/team/{team_id}/policies/delete` + +#### Parameters + +| Name | Type | In | Description | +| -------- | ------- | ---- | ------------------------------------------------- | +| team_id | integer | url | Defines what team id to operate on | +| ids | list | body | **Required.** The IDs of the policies to delete. | + +#### Example + +`POST /api/v1/fleet/global/policies/delete` + +#### Request body + +``` +{ + "ids": [ 1 ] +} +``` + +##### Default response + +`Status: 200` + +``` +{ + "deleted": 1 +} +``` + +--- + ## Activities ### List activities diff --git a/server/authz/authz.go b/server/authz/authz.go index 91e8895e4b..6402512253 100644 --- a/server/authz/authz.go +++ b/server/authz/authz.go @@ -119,6 +119,37 @@ func (a *Authorizer) Authorize(ctx context.Context, object, action interface{}) return nil } +func (a *Authorizer) TeamAuthorize(ctx context.Context, teamID uint, action string) error { + subject := UserFromContext(ctx) + if subject == nil { + return ForbiddenWithInternal("nil subject always forbidden", subject, nil, action) + } + + // global admins and maintainers are authorized to work with teams + if subject.GlobalRole != nil { + switch *subject.GlobalRole { + case fleet.RoleAdmin, fleet.RoleMaintainer: + return nil + } + } + + for _, team := range subject.Teams { + if teamID == team.ID { + switch action { + case fleet.ActionWrite: + if team.Role == fleet.RoleMaintainer { + return nil + } + return ForbiddenWithInternal("team observer cannot write", subject, nil, action) + default: + return nil + } + } + } + + return ForbiddenWithInternal("not a member of the team", subject, nil, action) +} + // AuthzTyper is the interface that may be implemented to get a `type` // property added during marshaling for authorization. Any struct that will be // used as a subject or object in authorization should implement this interface. diff --git a/server/authz/policy.rego b/server/authz/policy.rego index 9ff87d8780..64e5179104 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -347,6 +347,13 @@ allow { action == [read, write][_] } +# Team Maintainers can read and write policies +allow { + object.type == "policy" + team_role(subject, subject.teams[_].id) == maintainer + action == [read, write][_] +} + ## # Software ## diff --git a/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go b/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go new file mode 100644 index 0000000000..40cec1783a --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210915144307_AddPoliciesTeamColumn.go @@ -0,0 +1,25 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210915144307, Down_20210915144307) +} + +func Up_20210915144307(tx *sql.Tx) error { + if _, err := tx.Exec(`ALTER TABLE policies + ADD COLUMN team_id INT UNSIGNED, + ADD FOREIGN KEY fk_policies_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE + `); err != nil { + return errors.Wrap(err, "add column team_id") + } + return nil +} + +func Down_20210915144307(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 256d11a1f5..1b0a196d7e 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -22,23 +22,30 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, queryID uint) (*fleet. return nil, errors.Wrap(err, "getting last id after inserting policy") } - return policyDB(ctx, ds.writer, uint(lastIdInt64)) + return policyDB(ctx, ds.writer, uint(lastIdInt64), nil) } func (ds *Datastore) Policy(ctx context.Context, id uint) (*fleet.Policy, error) { - return policyDB(ctx, ds.reader, id) + return policyDB(ctx, ds.reader, id, nil) } -func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint) (*fleet.Policy, error) { +func policyDB(ctx context.Context, q sqlx.QueryerContext, id uint, teamID *uint) (*fleet.Policy, error) { + teamWhere := "TRUE" + args := []interface{}{id} + if teamID != nil { + teamWhere = "team_id = ?" + args = append(args, *teamID) + } + var policy fleet.Policy err := sqlx.GetContext(ctx, q, &policy, - `SELECT + fmt.Sprintf(`SELECT p.*, q.name as query_name, (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count - FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE p.id=?`, - id) + FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE p.id=? AND %s`, teamWhere), + args...) if err != nil { return nil, errors.Wrap(err, "getting policy") } @@ -78,17 +85,27 @@ func (ds *Datastore) RecordPolicyQueryExecutions(ctx context.Context, host *flee } func (ds *Datastore) ListGlobalPolicies(ctx context.Context) ([]*fleet.Policy, error) { + return listPoliciesDB(ctx, ds.reader, nil) +} + +func listPoliciesDB(ctx context.Context, q sqlx.QueryerContext, teamID *uint) ([]*fleet.Policy, error) { + teamWhere := "p.team_id is NULL" + var args []interface{} + if teamID != nil { + teamWhere = "p.team_id = ?" + args = append(args, *teamID) + } var policies []*fleet.Policy err := sqlx.SelectContext( ctx, - ds.reader, + q, &policies, - `SELECT + fmt.Sprintf(`SELECT p.*, q.name as query_name, (select count(*) from policy_membership where policy_id=p.id and passes=true) as passing_host_count, (select count(*) from policy_membership where policy_id=p.id and passes=false) as failing_host_count - FROM policies p JOIN queries q ON (p.query_id=q.id)`, + FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE %s`, teamWhere), args..., ) if err != nil { return nil, errors.Wrap(err, "listing policies") @@ -97,33 +114,91 @@ func (ds *Datastore) ListGlobalPolicies(ctx context.Context) ([]*fleet.Policy, e } func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { - stmt := `DELETE FROM policies WHERE id IN (?)` + return deletePolicyDB(ctx, ds.writer, ids, nil) +} + +func deletePolicyDB(ctx context.Context, q sqlx.ExtContext, ids []uint, teamID *uint) ([]uint, error) { + stmt := `DELETE FROM policies WHERE id IN (?) AND %s` stmt, args, err := sqlx.In(stmt, ids) if err != nil { return nil, errors.Wrap(err, "IN for DELETE FROM policies") } - stmt = ds.writer.Rebind(stmt) - if _, err := ds.writer.ExecContext(ctx, stmt, args...); err != nil { + stmt = q.Rebind(stmt) + + teamWhere := "TRUE" + if teamID != nil { + teamWhere = "team_id = ?" + args = append(args, *teamID) + } + + if _, err := q.ExecContext(ctx, fmt.Sprintf(stmt, teamWhere), args...); err != nil { return nil, errors.Wrap(err, "delete policies") } return ids, nil } -func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, _ *fleet.Host) (map[string]string, error) { - var rows []struct { +func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { + var globalRows, teamRows []struct { Id string `db:"id"` Query string `db:"query"` } - err := sqlx.SelectContext(ctx, ds.reader, &rows, `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id)`) + err := sqlx.SelectContext( + ctx, + ds.reader, + &globalRows, + `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE team_id is NULL`, + ) if err != nil { return nil, errors.Wrap(err, "selecting policies for host") } results := map[string]string{} - for _, row := range rows { + if host.TeamID != nil { + err := sqlx.SelectContext( + ctx, + ds.reader, + &teamRows, + `SELECT p.id, q.query FROM policies p JOIN queries q ON (p.query_id=q.id) WHERE team_id = ?`, + *host.TeamID, + ) + if err != nil { + return nil, errors.Wrap(err, "selecting policies for host in team") + } + } + + for _, row := range globalRows { + results[row.Id] = row.Query + } + + for _, row := range teamRows { results[row.Id] = row.Query } return results, nil } + +func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + res, err := ds.writer.ExecContext(ctx, `INSERT INTO policies (query_id, team_id) VALUES (?, ?)`, queryID, teamID) + if err != nil { + return nil, errors.Wrap(err, "inserting new team policy") + } + lastIdInt64, err := res.LastInsertId() + if err != nil { + return nil, errors.Wrap(err, "getting last id after inserting policy") + } + + return policyDB(ctx, ds.writer, uint(lastIdInt64), nil) +} + +func (ds *Datastore) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + return listPoliciesDB(ctx, ds.reader, &teamID) +} + +func (ds *Datastore) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + return deletePolicyDB(ctx, ds.writer, ids, &teamID) +} + +func (ds *Datastore) TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + return policyDB(ctx, ds.reader, policyID, &teamID) +} diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index fca687aa4c..a5fb49b7eb 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -142,9 +142,136 @@ func TestPolicyMembershipView(t *testing.T) { require.NoError(t, err) assert.Equal(t, policies[0], policy) - queries, err := ds.PolicyQueriesForHost(context.Background(), nil) + queries, err := ds.PolicyQueriesForHost(context.Background(), host1) require.NoError(t, err) require.Len(t, queries, 2) assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) assert.Equal(t, q2.Query, queries[fmt.Sprint(q2.ID)]) } + +func TestTeamPolicy(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + q, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Description: "query1 desc", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + + team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) + require.NoError(t, err) + + q2, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Description: "query2 desc", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + + prevPolicies, err := ds.ListGlobalPolicies(context.Background()) + require.NoError(t, err) + + _, err = ds.NewTeamPolicy(context.Background(), 99999999, q.ID) + require.Error(t, err) + + p, err := ds.NewTeamPolicy(context.Background(), team1.ID, q.ID) + require.NoError(t, err) + + assert.Equal(t, "query1", p.QueryName) + + globalPolicies, err := ds.ListGlobalPolicies(context.Background()) + require.NoError(t, err) + require.Len(t, globalPolicies, len(prevPolicies)) + + _, err = ds.NewTeamPolicy(context.Background(), team2.ID, q2.ID) + require.NoError(t, err) + + teamPolicies, err := ds.ListTeamPolicies(context.Background(), team1.ID) + require.NoError(t, err) + require.Len(t, teamPolicies, 1) + assert.Equal(t, q.ID, teamPolicies[0].QueryID) + + team2Policies, err := ds.ListTeamPolicies(context.Background(), team2.ID) + require.NoError(t, err) + require.Len(t, team2Policies, 1) + assert.Equal(t, q2.ID, team2Policies[0].QueryID) + + _, err = ds.DeleteTeamPolicies(context.Background(), team1.ID, []uint{teamPolicies[0].ID}) + require.NoError(t, err) + + teamPolicies, err = ds.ListGlobalPolicies(context.Background()) + require.NoError(t, err) + require.Len(t, teamPolicies, 0) +} + +func TestPolicyQueriesForHost(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + + team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) + require.NoError(t, err) + + host1, err := ds.NewHost(context.Background(), &fleet.Host{ + OsqueryHostID: "1234", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + }) + require.NoError(t, err) + + require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID})) + host1, err = ds.Host(context.Background(), host1.ID) + require.NoError(t, err) + + host2, err := ds.NewHost(context.Background(), &fleet.Host{ + OsqueryHostID: "5679", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "2", + UUID: "2", + Hostname: "bar.local", + }) + require.NoError(t, err) + + q, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Description: "query1 desc", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + _, err = ds.NewGlobalPolicy(context.Background(), q.ID) + require.NoError(t, err) + + q2, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Description: "query2 desc", + Query: "select 42;", + Saved: true, + }) + require.NoError(t, err) + _, err = ds.NewTeamPolicy(context.Background(), team1.ID, q2.ID) + require.NoError(t, err) + + queries, err := ds.PolicyQueriesForHost(context.Background(), host1) + require.NoError(t, err) + require.Len(t, queries, 2) + assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) + assert.Equal(t, q2.Query, queries[fmt.Sprint(q2.ID)]) + + queries, err = ds.PolicyQueriesForHost(context.Background(), host2) + require.NoError(t, err) + require.Len(t, queries, 1) + assert.Equal(t, q.Query, queries[fmt.Sprint(q.ID)]) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 8db6f3112e..2ffbec1364 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -310,9 +310,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=102 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=103 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -400,9 +400,12 @@ CREATE TABLE `policies` ( `query_id` int(10) unsigned NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `team_id` int(10) unsigned DEFAULT NULL, PRIMARY KEY (`id`), KEY `fk_policies_query_id` (`query_id`), - CONSTRAINT `policies_ibfk_1` FOREIGN KEY (`query_id`) REFERENCES `queries` (`id`) + KEY `fk_policies_team_id` (`team_id`), + CONSTRAINT `policies_ibfk_1` FOREIGN KEY (`query_id`) REFERENCES `queries` (`id`), + CONSTRAINT `policies_ibfk_2` FOREIGN KEY (`team_id`) REFERENCES `teams` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; SET @saved_cs_client = @@character_set_client; diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index c333c010a4..24b904cc24 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -331,7 +331,8 @@ type Datastore interface { RecordStatisticsSent(ctx context.Context) error /////////////////////////////////////////////////////////////////////////////// - // GlobalPoliciesStore interface { + // GlobalPoliciesStore + NewGlobalPolicy(ctx context.Context, queryID uint) (*Policy, error) Policy(ctx context.Context, id uint) (*Policy, error) RecordPolicyQueryExecutions(ctx context.Context, host *Host, results map[uint]*bool, updated time.Time) error @@ -349,6 +350,14 @@ type Datastore interface { MigrationStatus(ctx context.Context) (MigrationStatus, error) ListSoftware(ctx context.Context, teamId *uint, opt ListOptions) ([]Software, error) + + /////////////////////////////////////////////////////////////////////////////// + // Team Policies + + NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*Policy, error) + ListTeamPolicies(ctx context.Context, teamID uint) ([]*Policy, error) + DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) + TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*Policy, error) } type MigrationStatus int diff --git a/server/fleet/global_policies.go b/server/fleet/global_policies.go index e41bacb87d..d84f594c45 100644 --- a/server/fleet/global_policies.go +++ b/server/fleet/global_policies.go @@ -6,6 +6,7 @@ type Policy struct { QueryName string `json:"query_name" db:"query_name"` PassingHostCount uint `json:"passing_host_count" db:"passing_host_count"` FailingHostCount uint `json:"failing_host_count" db:"failing_host_count"` + TeamID *uint `db:"team_id"` UpdateCreateTimestamps } diff --git a/server/fleet/service.go b/server/fleet/service.go index 3f4e8c2b91..8c7f4568be 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -358,6 +358,7 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // ActivitiesService + ListActivities(ctx context.Context, opt ListOptions) ([]*Activity, error) /////////////////////////////////////////////////////////////////////////////// @@ -368,6 +369,7 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // GlobalScheduleService + GlobalScheduleQuery(ctx context.Context, sq *ScheduledQuery) (*ScheduledQuery, error) GetGlobalScheduledQueries(ctx context.Context, opts ListOptions) ([]*ScheduledQuery, error) ModifyGlobalScheduledQueries(ctx context.Context, id uint, q ScheduledQueryPayload) (*ScheduledQuery, error) @@ -375,10 +377,12 @@ type Service interface { /////////////////////////////////////////////////////////////////////////////// // TranslatorService + Translate(ctx context.Context, payloads []TranslatePayload) ([]TranslatePayload, error) /////////////////////////////////////////////////////////////////////////////// // TeamScheduleService + TeamScheduleQuery(ctx context.Context, teamID uint, sq *ScheduledQuery) (*ScheduledQuery, error) GetTeamScheduledQueries(ctx context.Context, teamID uint, opts ListOptions) ([]*ScheduledQuery, error) ModifyTeamScheduledQueries( @@ -398,4 +402,12 @@ type Service interface { // Software ListSoftware(ctx context.Context, teamID *uint, opt ListOptions) ([]Software, error) + + /////////////////////////////////////////////////////////////////////////////// + // Team Policies + + NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*Policy, error) + ListTeamPolicies(ctx context.Context, teamID uint) ([]*Policy, error) + DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) + GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*Policy, error) } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 666db34709..948b9df08c 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -277,6 +277,14 @@ type MigrationStatusFunc func(ctx context.Context) (fleet.MigrationStatus, error type ListSoftwareFunc func(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) +type NewTeamPolicyFunc func(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) + +type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) + +type DeleteTeamPoliciesFunc func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) + +type TeamPolicyFunc func(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) + type DataStore struct { NewCarveFunc NewCarveFunc NewCarveFuncInvoked bool @@ -676,6 +684,18 @@ type DataStore struct { ListSoftwareFunc ListSoftwareFunc ListSoftwareFuncInvoked bool + + NewTeamPolicyFunc NewTeamPolicyFunc + NewTeamPolicyFuncInvoked bool + + ListTeamPoliciesFunc ListTeamPoliciesFunc + ListTeamPoliciesFuncInvoked bool + + DeleteTeamPoliciesFunc DeleteTeamPoliciesFunc + DeleteTeamPoliciesFuncInvoked bool + + TeamPolicyFunc TeamPolicyFunc + TeamPolicyFuncInvoked bool } func (s *DataStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { @@ -1342,3 +1362,23 @@ func (s *DataStore) ListSoftware(ctx context.Context, teamId *uint, opt fleet.Li s.ListSoftwareFuncInvoked = true return s.ListSoftwareFunc(ctx, teamId, opt) } + +func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + s.NewTeamPolicyFuncInvoked = true + return s.NewTeamPolicyFunc(ctx, teamID, queryID) +} + +func (s *DataStore) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + s.ListTeamPoliciesFuncInvoked = true + return s.ListTeamPoliciesFunc(ctx, teamID) +} + +func (s *DataStore) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + s.DeleteTeamPoliciesFuncInvoked = true + return s.DeleteTeamPoliciesFunc(ctx, teamID, ids) +} + +func (s *DataStore) TeamPolicy(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + s.TeamPolicyFuncInvoked = true + return s.TeamPolicyFunc(ctx, teamID, policyID) +} diff --git a/server/service/handler.go b/server/service/handler.go index 90518ab52b..f6c894977c 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -705,6 +705,11 @@ func attachNewStyleFleetAPIRoutes(r *mux.Router, svc fleet.Service, opts []kitht e.GET("/api/v1/fleet/global/policies/{policy_id}", getPolicyByIDEndpoint, getPolicyByIDRequest{}) e.POST("/api/v1/fleet/global/policies/delete", deleteGlobalPoliciesEndpoint, deleteGlobalPoliciesRequest{}) + e.POST("/api/v1/fleet/team/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) + e.GET("/api/v1/fleet/team/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) + e.GET("/api/v1/fleet/team/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) + e.POST("/api/v1/fleet/team/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) + e.GET("/api/v1/fleet/software", listSoftwareEndpoint, listSoftwareRequest{}) } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 53af9f01b6..a6b7ea9b14 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -106,7 +106,10 @@ func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 0) - qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery2", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) + qr, err := s.ds.NewQuery( + context.Background(), + &fleet.Query{Name: "TestQueryTeamPolicy", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}, + ) require.NoError(t, err) gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} @@ -117,7 +120,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 1) assert.Equal(t, uint(42), ts.Scheduled[0].Interval) - assert.Equal(t, "TestQuery2", ts.Scheduled[0].Name) + assert.Equal(t, "TestQueryTeamPolicy", ts.Scheduled[0].Name) assert.Equal(t, qr.ID, ts.Scheduled[0].QueryID) id := ts.Scheduled[0].ID @@ -140,3 +143,64 @@ func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 0) } + +func (s *integrationEnterpriseTestSuite) TestTeamPolicies() { + t := s.T() + + team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + ID: 42, + Name: "team1" + t.Name(), + Description: "desc team1", + }) + require.NoError(t, err) + + oldToken := s.token + t.Cleanup(func() { + s.token = oldToken + }) + + password := "garbage" + email := "testteam@user.com" + + u := &fleet.User{ + Name: "test team user", + Email: email, + GlobalRole: nil, + Teams: []fleet.UserTeam{ + { + Team: *team1, + Role: fleet.RoleMaintainer, + }, + }, + } + require.NoError(t, u.SetPassword(password, 10, 10)) + _, err = s.ds.NewUser(context.Background(), u) + require.NoError(t, err) + + s.token = s.getTestToken(email, password) + + ts := listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 0) + + qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery2", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) + require.NoError(t, err) + + tpParams := teamPolicyRequest{QueryID: qr.ID} + r := teamPolicyResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), tpParams, http.StatusOK, &r) + + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 1) + assert.Equal(t, "TestQuery2", ts.Policies[0].QueryName) + assert.Equal(t, qr.ID, ts.Policies[0].QueryID) + + deletePolicyParams := deleteTeamPoliciesRequest{IDs: []uint{ts.Policies[0].ID}} + deletePolicyResp := deleteTeamPoliciesResponse{} + s.DoJSON("POST", "/api/v1/fleet/global/policies/delete", deletePolicyParams, http.StatusOK, &deletePolicyResp) + + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 0) +} diff --git a/server/service/team_policies.go b/server/service/team_policies.go new file mode 100644 index 0000000000..3ac3822345 --- /dev/null +++ b/server/service/team_policies.go @@ -0,0 +1,155 @@ +package service + +import ( + "context" + + "github.com/fleetdm/fleet/v4/server/fleet" +) + +///////////////////////////////////////////////////////////////////////////////// +// Add +///////////////////////////////////////////////////////////////////////////////// + +type teamPolicyRequest struct { + TeamID uint `url:"team_id"` + QueryID uint `json:"query_id"` +} + +type teamPolicyResponse struct { + Policy *fleet.Policy `json:"policy,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r teamPolicyResponse) error() error { return r.Err } + +func teamPolicyEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*teamPolicyRequest) + resp, err := svc.NewTeamPolicy(ctx, req.TeamID, req.QueryID) + if err != nil { + return teamPolicyResponse{Err: err}, nil + } + return teamPolicyResponse{Policy: resp}, nil +} + +func (svc Service) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionWrite); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return nil, err + } + + return svc.ds.NewTeamPolicy(ctx, teamID, queryID) +} + +///////////////////////////////////////////////////////////////////////////////// +// List +///////////////////////////////////////////////////////////////////////////////// + +type listTeamPoliciesRequest struct { + TeamID uint `url:"team_id"` +} + +type listTeamPoliciesResponse struct { + Policies []*fleet.Policy `json:"policies,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r listTeamPoliciesResponse) error() error { return r.Err } + +func listTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*listTeamPoliciesRequest) + resp, err := svc.ListTeamPolicies(ctx, req.TeamID) + if err != nil { + return listTeamPoliciesResponse{Err: err}, nil + } + return listTeamPoliciesResponse{Policies: resp}, nil +} + +func (svc Service) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionRead); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } + + return svc.ds.ListTeamPolicies(ctx, teamID) +} + +///////////////////////////////////////////////////////////////////////////////// +// Get by id +///////////////////////////////////////////////////////////////////////////////// + +type getTeamPolicyByIDRequest struct { + TeamID uint `url:"team_id"` + PolicyID uint `url:"policy_id"` +} + +type getTeamPolicyByIDResponse struct { + Policy *fleet.Policy `json:"policy"` + Err error `json:"error,omitempty"` +} + +func (r getTeamPolicyByIDResponse) error() error { return r.Err } + +func getTeamPolicyByIDEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*getTeamPolicyByIDRequest) + teamPolicy, err := svc.GetTeamPolicyByIDQueries(ctx, req.TeamID, req.PolicyID) + if err != nil { + return getTeamPolicyByIDResponse{Err: err}, nil + } + return getTeamPolicyByIDResponse{Policy: teamPolicy}, nil +} + +func (svc Service) GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionRead); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } + + teamPolicy, err := svc.ds.TeamPolicy(ctx, teamID, policyID) + if err != nil { + return nil, err + } + + return teamPolicy, nil +} + +///////////////////////////////////////////////////////////////////////////////// +// Delete +///////////////////////////////////////////////////////////////////////////////// + +type deleteTeamPoliciesRequest struct { + TeamID uint `url:"team_id"` + IDs []uint `json:"ids"` +} + +type deleteTeamPoliciesResponse struct { + Deleted []uint `json:"deleted,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r deleteTeamPoliciesResponse) error() error { return r.Err } + +func deleteTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) { + req := request.(*deleteTeamPoliciesRequest) + resp, err := svc.DeleteTeamPolicies(ctx, req.TeamID, req.IDs) + if err != nil { + return deleteTeamPoliciesResponse{Err: err}, nil + } + return deleteTeamPoliciesResponse{Deleted: resp}, nil +} + +func (svc Service) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionWrite); err != nil { + return nil, err + } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return nil, err + } + + return svc.ds.DeleteTeamPolicies(ctx, teamID, ids) +} diff --git a/server/service/team_schedule.go b/server/service/team_schedule.go index f1578d966f..9561cd2595 100644 --- a/server/service/team_schedule.go +++ b/server/service/team_schedule.go @@ -39,6 +39,9 @@ func (svc Service) GetTeamScheduledQueries(ctx context.Context, teamID uint, opt if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionRead); err != nil { return nil, err } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -102,6 +105,9 @@ func (svc Service) TeamScheduleQuery(ctx context.Context, teamID uint, q *fleet. if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionRead); err != nil { return nil, err } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { + return nil, err + } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -143,6 +149,9 @@ func (svc Service) ModifyTeamScheduledQueries(ctx context.Context, teamID uint, if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionWrite); err != nil { return nil, err } + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return nil, err + } gp, err := svc.ds.EnsureTeamPack(ctx, teamID) if err != nil { @@ -183,6 +192,8 @@ func (svc Service) DeleteTeamScheduledQueries(ctx context.Context, teamID uint, if err := svc.authz.Authorize(ctx, &fleet.Pack{}, fleet.ActionWrite); err != nil { return err } - _ = teamID + if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { + return err + } return svc.DeleteScheduledQuery(ctx, scheduledQueryID) } diff --git a/server/service/testing_client.go b/server/service/testing_client.go index e5c1e3be7b..89cbaf28eb 100644 --- a/server/service/testing_client.go +++ b/server/service/testing_client.go @@ -104,9 +104,13 @@ func (ts *withServer) DoJSON(verb, path string, params interface{}, expectedStat func (ts *withServer) getTestAdminToken() string { testUser := testUsers["admin1"] + return ts.getTestToken(testUser.Email, testUser.PlaintextPassword) +} + +func (ts *withServer) getTestToken(email string, password string) string { params := loginRequest{ - Email: testUser.Email, - Password: testUser.PlaintextPassword, + Email: email, + Password: password, } j, err := json.Marshal(¶ms) require.NoError(ts.s.T(), err) @@ -123,7 +127,8 @@ func (ts *withServer) getTestAdminToken() string { Err []map[string]string `json:"errors,omitempty"` }{} err = json.NewDecoder(resp.Body).Decode(&jsn) - require.Nil(ts.s.T(), err) + require.NoError(ts.s.T(), err) + require.Len(ts.s.T(), jsn.Err, 0) return jsn.Token } From e286ee387ecb987ea402c86bda8084e023975727 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 20 Sep 2021 11:09:51 -0300 Subject: [PATCH 23/82] Allow team maintainers to run new queries in the team hosts (#2076) * Allow team maintainers to run new queries in the team hosts * Add policies for other roles --- changes/issue-2062-team-maintainer-run-new | 1 + server/authz/policy.rego | 27 ++++++++++ server/fleet/authz.go | 2 + server/service/service_campaigns.go | 2 +- server/service/service_osquery_test.go | 57 ++++++++++++++++++++++ 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 changes/issue-2062-team-maintainer-run-new diff --git a/changes/issue-2062-team-maintainer-run-new b/changes/issue-2062-team-maintainer-run-new new file mode 100644 index 0000000000..eed0492a76 --- /dev/null +++ b/changes/issue-2062-team-maintainer-run-new @@ -0,0 +1 @@ +* Allow team maintainers to run new queries in the team hosts. diff --git a/server/authz/policy.rego b/server/authz/policy.rego index 64e5179104..64ccb76a09 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -15,6 +15,7 @@ list := "list" write := "write" write_role := "write_role" run := "run" +run_new := "run_new" # Roles admin := "admin" @@ -265,6 +266,16 @@ allow { subject.global_role == maintainer action = run } +allow { + object.type == "query" + subject.global_role == admin + action = run_new +} +allow { + object.type == "query" + subject.global_role == maintainer + action = run_new +} # Team maintainer running a non-observers_can_run query must have the targets # filtered to only teams that they maintain allow { @@ -274,6 +285,22 @@ allow { action == run } +# Team maintainer can run a new query +allow { + object.type == "query" + # If role is maintainer on any team + team_role(subject, subject.teams[_].id) == maintainer + action == run_new +} + +# Team admin can run a new query +allow { + object.type == "query" + # If role is maintainer on any team + team_role(subject, subject.teams[_].id) == admin + action == run_new +} + # (Team) observers can run only if observers_can_run allow { object.type == "query" diff --git a/server/fleet/authz.go b/server/fleet/authz.go index 94cda02869..118dcce871 100644 --- a/server/fleet/authz.go +++ b/server/fleet/authz.go @@ -11,4 +11,6 @@ const ( ActionWriteRole = "write_role" // ActionRun is the action for running a live query. ActionRun = "run" + // ActionRunNew is the action for running a new live query. + ActionRunNew = "run_new" ) diff --git a/server/service/service_campaigns.go b/server/service/service_campaigns.go index 0802a2f99c..ec1cd90e17 100644 --- a/server/service/service_campaigns.go +++ b/server/service/service_campaigns.go @@ -61,7 +61,7 @@ func (svc Service) NewDistributedQueryCampaign(ctx context.Context, queryString } queryString = query.Query } else { - if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionWrite); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionRunNew); err != nil { return nil, err } query = &fleet.Query{ diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 30d7b9fa56..00570228cb 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -1746,6 +1746,63 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) { require.NoError(t, err) } +func TestTeamMaintainerCanRunNewDistributedCampaigns(t *testing.T) { + ds := new(mock.Store) + rs := &mock.QueryResultStore{ + HealthCheckFunc: func() error { + return nil + }, + } + lq := &live_query.MockLiveQuery{} + mockClock := clock.NewMockClock() + svc := newTestServiceWithClock(ds, rs, lq, mockClock) + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{}, nil + } + + ds.NewDistributedQueryCampaignFunc = func(ctx context.Context, camp *fleet.DistributedQueryCampaign) (*fleet.DistributedQueryCampaign, error) { + return camp, nil + } + ds.QueryFunc = func(ctx context.Context, id uint) (*fleet.Query, error) { + return &fleet.Query{ + ID: 42, + Name: "query", + Query: "select 1;", + ObserverCanRun: false, + }, nil + } + viewerCtx := viewer.NewContext(context.Background(), viewer.Viewer{ + User: &fleet.User{ID: 0, Teams: []fleet.UserTeam{{Role: fleet.RoleMaintainer}}}, + }) + + q := "select year, month, day, hour, minutes, seconds from time" + ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { + return nil + } + //var gotQuery *fleet.Query + ds.NewQueryFunc = func(ctx context.Context, query *fleet.Query, opts ...fleet.OptionalArg) (*fleet.Query, error) { + //gotQuery = query + query.ID = 42 + return query, nil + } + ds.NewDistributedQueryCampaignTargetFunc = func(ctx context.Context, target *fleet.DistributedQueryCampaignTarget) (*fleet.DistributedQueryCampaignTarget, error) { + return target, nil + } + ds.CountHostsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets, now time.Time) (fleet.TargetMetrics, error) { + return fleet.TargetMetrics{}, nil + } + ds.HostIDsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets) ([]uint, error) { + return []uint{1, 3, 5}, nil + } + ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { + return nil + } + lq.On("RunQuery", "0", "select year, month, day, hour, minutes, seconds from time", []uint{1, 3, 5}).Return(nil) + _, err := svc.NewDistributedQueryCampaign(viewerCtx, q, nil, fleet.HostTargets{HostIDs: []uint{2}, LabelIDs: []uint{1}}) + require.NoError(t, err) +} + func TestPolicyQueries(t *testing.T) { mockClock := clock.NewMockClock() ds := new(mock.Store) From b32b441c12ed13f0d53318bdf1937ff956ae6c52 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 20 Sep 2021 13:07:51 -0300 Subject: [PATCH 24/82] Issue 1512 filter observer can run queries (#2110) * wip * Filter queries for observers * Update e2e test now that we filter queries --- changes/issue-1512-filter-queries | 1 + cypress/integration/free/observer.spec.ts | 9 +-- server/datastore/mysql/queries.go | 8 ++- server/datastore/mysql/queries_test.go | 77 +++++++++++++++++------ server/fleet/app.go | 6 ++ server/fleet/datastore.go | 2 +- server/mock/datastore_mock.go | 4 +- server/service/service_queries.go | 31 ++++++++- server/service/service_queries_test.go | 61 ++++++++++++++++++ 9 files changed, 164 insertions(+), 35 deletions(-) create mode 100644 changes/issue-1512-filter-queries diff --git a/changes/issue-1512-filter-queries b/changes/issue-1512-filter-queries new file mode 100644 index 0000000000..4ed072c52b --- /dev/null +++ b/changes/issue-1512-filter-queries @@ -0,0 +1 @@ +* Only show observers queries they can run. diff --git a/cypress/integration/free/observer.spec.ts b/cypress/integration/free/observer.spec.ts index ab7e05af38..b020bfd349 100644 --- a/cypress/integration/free/observer.spec.ts +++ b/cypress/integration/free/observer.spec.ts @@ -67,14 +67,7 @@ describe("Free tier - Observer user", () => { cy.visit("/queries/manage"); - cy.findByText(/get authorized/i).click(); - cy.findByText(/packs/i).should("not.exist"); - cy.findByLabelText(/query name/i).should("not.exist"); - cy.findByLabelText(/sql/i).should("not.exist"); - cy.findByLabelText(/description/i).should("not.exist"); - cy.findByLabelText(/observer can run/i).should("not.exist"); - cy.findByText(/show sql/i).click(); - cy.findByRole("button", { name: /run query/i }).should("not.exist"); + cy.findByText(/get authorized/i).should("not.exist"); // On the Profile page, they should… // See Observer in Role section, and no Team section diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index 56c98ad104..e15248ce9d 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -170,7 +170,7 @@ func (d *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { // ListQueries returns a list of queries with sort order and results limit // determined by passed in fleet.ListOptions -func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { +func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { sql := ` SELECT q.*, COALESCE(u.name, '') AS author_name FROM queries q @@ -178,7 +178,11 @@ func (d *Datastore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]* ON q.author_id = u.id WHERE saved = true ` - sql = appendListOptionsToSQL(sql, opt) + if opt.OnlyObserverCanRun { + sql += " AND q.observer_can_run=true" + } + sql = appendListOptionsToSQL(sql, opt.ListOptions) + results := []*fleet.Query{} if err := sqlx.SelectContext(ctx, d.reader, &results, sql); err != nil { diff --git a/server/datastore/mysql/queries_test.go b/server/datastore/mysql/queries_test.go index 593193c8b7..c080289a81 100644 --- a/server/datastore/mysql/queries_test.go +++ b/server/datastore/mysql/queries_test.go @@ -29,7 +29,7 @@ func TestApplyQueries(t *testing.T) { err := ds.ApplyQueries(context.Background(), zwass.ID, expectedQueries) require.Nil(t, err) - queries, err := ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -47,7 +47,7 @@ func TestApplyQueries(t *testing.T) { err = ds.ApplyQueries(context.Background(), groob.ID, expectedQueries) require.Nil(t, err) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -65,7 +65,7 @@ func TestApplyQueries(t *testing.T) { err = ds.ApplyQueries(context.Background(), zwass.ID, []*fleet.Query{expectedQueries[2]}) require.Nil(t, err) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) require.Len(t, queries, len(expectedQueries)) for i, q := range queries { @@ -130,7 +130,7 @@ func TestDeleteQueries(t *testing.T) { q3 := test.NewQuery(t, ds, "q3", "select 1", user.ID, true) q4 := test.NewQuery(t, ds, "q4", "select * from osquery_info", user.ID, true) - queries, err := ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 4) @@ -138,7 +138,7 @@ func TestDeleteQueries(t *testing.T) { require.Nil(t, err) assert.Equal(t, uint(2), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 2) @@ -146,7 +146,7 @@ func TestDeleteQueries(t *testing.T) { require.Nil(t, err) assert.Equal(t, uint(1), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 1) @@ -154,7 +154,7 @@ func TestDeleteQueries(t *testing.T) { require.Nil(t, err) assert.Equal(t, uint(1), deleted) - queries, err = ds.ListQueries(context.Background(), fleet.ListOptions{}) + queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 0) @@ -215,7 +215,7 @@ func TestListQuery(t *testing.T) { }) require.Nil(t, err) - opts := fleet.ListOptions{} + opts := fleet.ListQueryOptions{} results, err := ds.ListQueries(context.Background(), opts) assert.Nil(t, err) assert.Equal(t, 10, len(results)) @@ -234,9 +234,9 @@ func TestLoadPacksForQueries(t *testing.T) { require.Nil(t, err) specs := []*fleet.PackSpec{ - &fleet.PackSpec{Name: "p1"}, - &fleet.PackSpec{Name: "p2"}, - &fleet.PackSpec{Name: "p3"}, + {Name: "p1"}, + {Name: "p2"}, + {Name: "p3"}, } err = ds.ApplyPackSpecs(context.Background(), specs) require.Nil(t, err) @@ -250,10 +250,10 @@ func TestLoadPacksForQueries(t *testing.T) { assert.Empty(t, q1.Packs) specs = []*fleet.PackSpec{ - &fleet.PackSpec{ + { Name: "p2", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { Name: "q0", QueryName: queries[0].Name, Interval: 60, @@ -275,19 +275,19 @@ func TestLoadPacksForQueries(t *testing.T) { assert.Empty(t, q1.Packs) specs = []*fleet.PackSpec{ - &fleet.PackSpec{ + { Name: "p1", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { QueryName: queries[1].Name, Interval: 60, }, }, }, - &fleet.PackSpec{ + { Name: "p3", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { QueryName: queries[1].Name, Interval: 60, }, @@ -312,15 +312,15 @@ func TestLoadPacksForQueries(t *testing.T) { } specs = []*fleet.PackSpec{ - &fleet.PackSpec{ + { Name: "p3", Queries: []fleet.PackSpecQuery{ - fleet.PackSpecQuery{ + { Name: "q0", QueryName: queries[0].Name, Interval: 60, }, - fleet.PackSpecQuery{ + { Name: "q1", QueryName: queries[1].Name, Interval: 60, @@ -370,3 +370,40 @@ func TestDuplicateNewQuery(t *testing.T) { // is private to the individual datastore implementations assert.Contains(t, err.Error(), "already exists") } + +func TestListQueryFiltersObserver(t *testing.T) { + ds := CreateMySQLDS(t) + defer ds.Close() + + _, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query1", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + _, err = ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query2", + Query: "select 1;", + Saved: true, + }) + require.NoError(t, err) + query3, err := ds.NewQuery(context.Background(), &fleet.Query{ + Name: "query3", + Query: "select 1;", + Saved: true, + ObserverCanRun: true, + }) + require.NoError(t, err) + + queries, err := ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) + require.NoError(t, err) + require.Len(t, queries, 3) + + queries, err = ds.ListQueries( + context.Background(), + fleet.ListQueryOptions{OnlyObserverCanRun: true, ListOptions: fleet.ListOptions{PerPage: 1}}, + ) + require.NoError(t, err) + require.Len(t, queries, 1) + assert.Equal(t, query3.ID, queries[0].ID) +} diff --git a/server/fleet/app.go b/server/fleet/app.go index 043dc0259d..3b141315f6 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -243,6 +243,12 @@ type ListOptions struct { MatchQuery string } +type ListQueryOptions struct { + ListOptions + + OnlyObserverCanRun bool +} + // EnrollSecret contains information about an enroll secret, name, and active // status. Enroll secrets are used for osquery authentication. type EnrollSecret struct { diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 24b904cc24..0949db4d32 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -61,7 +61,7 @@ type Datastore interface { Query(ctx context.Context, id uint) (*Query, error) // ListQueries returns a list of queries with the provided sorting and paging options. Associated packs should also // be loaded. - ListQueries(ctx context.Context, opt ListOptions) ([]*Query, error) + ListQueries(ctx context.Context, opt ListQueryOptions) ([]*Query, error) // QueryByName looks up a query by name. QueryByName(ctx context.Context, name string, opts ...OptionalArg) (*Query, error) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 948b9df08c..4c8218a9be 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -59,7 +59,7 @@ type DeleteQueriesFunc func(ctx context.Context, ids []uint) (uint, error) type QueryFunc func(ctx context.Context, id uint) (*fleet.Query, error) -type ListQueriesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) +type ListQueriesFunc func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) type QueryByNameFunc func(ctx context.Context, name string, opts ...fleet.OptionalArg) (*fleet.Query, error) @@ -818,7 +818,7 @@ func (s *DataStore) Query(ctx context.Context, id uint) (*fleet.Query, error) { return s.QueryFunc(ctx, id) } -func (s *DataStore) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { +func (s *DataStore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { s.ListQueriesFuncInvoked = true return s.ListQueriesFunc(ctx, opt) } diff --git a/server/service/service_queries.go b/server/service/service_queries.go index 65ea5722df..3eefd83733 100644 --- a/server/service/service_queries.go +++ b/server/service/service_queries.go @@ -66,7 +66,7 @@ func (svc Service) GetQuerySpecs(ctx context.Context) ([]*fleet.QuerySpec, error return nil, err } - queries, err := svc.ds.ListQueries(ctx, fleet.ListOptions{}) + queries, err := svc.ds.ListQueries(ctx, fleet.ListQueryOptions{}) if err != nil { return nil, errors.Wrap(err, "getting queries") } @@ -90,12 +90,39 @@ func (svc Service) GetQuerySpec(ctx context.Context, name string) (*fleet.QueryS return specFromQuery(query), nil } +func onlyShowObserverCanRunQueries(user *fleet.User) bool { + if user.GlobalRole != nil && *user.GlobalRole == fleet.RoleObserver { + return true + } else if len(user.Teams) > 0 { + allObserver := true + for _, team := range user.Teams { + if team.Role != fleet.RoleObserver { + allObserver = false + break + } + } + return allObserver + } + return false +} + func (svc Service) ListQueries(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Query, error) { if err := svc.authz.Authorize(ctx, &fleet.Query{}, fleet.ActionRead); err != nil { return nil, err } - return svc.ds.ListQueries(ctx, opt) + user := authz.UserFromContext(ctx) + onlyShowObserverCanRun := onlyShowObserverCanRunQueries(user) + + queries, err := svc.ds.ListQueries(ctx, fleet.ListQueryOptions{ + ListOptions: opt, + OnlyObserverCanRun: onlyShowObserverCanRun, + }) + if err != nil { + return nil, err + } + + return queries, nil } func (svc *Service) GetQuery(ctx context.Context, id uint) (*fleet.Query, error) { diff --git a/server/service/service_queries_test.go b/server/service/service_queries_test.go index 5167d6b6f7..5f3087a81a 100644 --- a/server/service/service_queries_test.go +++ b/server/service/service_queries_test.go @@ -4,8 +4,11 @@ import ( "context" "testing" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -21,3 +24,61 @@ func TestNewQueryAttach(t *testing.T) { ) require.Error(t, err) } + +func TestFilterQueriesForObserver(t *testing.T) { + require.True(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)})) + require.False(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)})) + require.False(t, onlyShowObserverCanRunQueries(&fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)})) + + require.True(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{{Role: fleet.RoleObserver}}})) + require.True(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{ + {Role: fleet.RoleObserver}, + {Role: fleet.RoleObserver}, + }})) + require.False(t, onlyShowObserverCanRunQueries(&fleet.User{Teams: []fleet.UserTeam{ + {Role: fleet.RoleObserver}, + {Role: fleet.RoleMaintainer}, + }})) +} + +func TestListQueries(t *testing.T) { + ds := new(mock.Store) + svc := newTestService(ds, nil, nil) + + cases := [...]struct { + title string + user *fleet.User + expectedOpts fleet.ListQueryOptions + }{ + { + title: "global admin", + user: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, + expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: false}, + }, + { + title: "global observer", + user: &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, + expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: true}, + }, + { + title: "team admin", + user: &fleet.User{Teams: []fleet.UserTeam{{Role: fleet.RoleAdmin}}}, + expectedOpts: fleet.ListQueryOptions{OnlyObserverCanRun: false}, + }, + } + + var calledWithOpts fleet.ListQueryOptions + ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) { + calledWithOpts = opt + return []*fleet.Query{}, nil + } + + for _, tt := range cases { + t.Run(tt.title, func(t *testing.T) { + viewerCtx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) + _, err := svc.ListQueries(viewerCtx, fleet.ListOptions{}) + require.NoError(t, err) + assert.Equal(t, tt.expectedOpts, calledWithOpts) + }) + } +} From eecef148eb54c805ab91f91459366fa6868da224 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 20 Sep 2021 14:46:51 -0300 Subject: [PATCH 25/82] Fail early if process does not have permissions to write to log file (#2138) * Fail early if process does not have permissions to write to log file * Open file once on NewFilesystemLogWriter --- .../issue-1950-logging-filesystem-fail-early | 1 + server/logging/filesystem.go | 71 ++++++++++--------- server/logging/filesystem_test.go | 23 ++++++ 3 files changed, 63 insertions(+), 32 deletions(-) create mode 100644 changes/issue-1950-logging-filesystem-fail-early diff --git a/changes/issue-1950-logging-filesystem-fail-early b/changes/issue-1950-logging-filesystem-fail-early new file mode 100644 index 0000000000..306f88436e --- /dev/null +++ b/changes/issue-1950-logging-filesystem-fail-early @@ -0,0 +1 @@ +* Fail early if the process does not have permissions to write to the logging file. diff --git a/server/logging/filesystem.go b/server/logging/filesystem.go index ac82ade98b..5b9a7e98bf 100644 --- a/server/logging/filesystem.go +++ b/server/logging/filesystem.go @@ -24,35 +24,42 @@ type filesystemLogWriter struct { // NewFilesystemLogWriter creates a log file for osquery status/result logs. // The logFile can be rotated by sending a `SIGHUP` signal to Fleet if // enableRotation is true +// +// The enableCompression argument is only used when enableRotation is true. func NewFilesystemLogWriter(path string, appLogger log.Logger, enableRotation bool, enableCompression bool) (*filesystemLogWriter, error) { - if enableRotation { - // Use lumberjack logger that supports rotation - osquerydLogger := &lumberjack.Logger{ - Filename: path, - MaxSize: 500, // megabytes - MaxBackups: 3, - MaxAge: 28, //days - Compress: enableCompression, - } - appLogger = log.With(appLogger, "component", "osqueryd-logger") - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGHUP) - go func() { - for { - <-sig //block on signal - if err := osquerydLogger.Rotate(); err != nil { - appLogger.Log("err", err) - } - } - }() - return &filesystemLogWriter{osquerydLogger}, nil - } - // no log rotation, use "raw" bufio implementation - writer, err := newRawLogWriter(path) + // Fail early if the process does not have the necessary + // permissions to open the file at path. + file, err := openFile(path) if err != nil { - return nil, errors.Wrap(err, "create new raw logger") + return nil, errors.Wrap(err, "perm check") } - return &filesystemLogWriter{writer}, nil + if !enableRotation { + // no log rotation, use "raw" bufio implementation + return &filesystemLogWriter{ + writer: newRawLogWriter(file), + }, nil + } + // Use lumberjack logger that supports rotation + file.Close() + osquerydLogger := &lumberjack.Logger{ + Filename: path, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: enableCompression, + } + appLogger = log.With(appLogger, "component", "osqueryd-logger") + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP) + go func() { + for { + <-sig //block on signal + if err := osquerydLogger.Rotate(); err != nil { + appLogger.Log("err", err) + } + } + }() + return &filesystemLogWriter{osquerydLogger}, nil } // If writer is based on bufio we want to flush after a batch of @@ -85,13 +92,9 @@ type rawLogWriter struct { mtx sync.Mutex } -func newRawLogWriter(path string) (*rawLogWriter, error) { - file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) - if err != nil { - return nil, err - } +func newRawLogWriter(file *os.File) *rawLogWriter { buff := bufio.NewWriter(file) - return &rawLogWriter{file: file, buff: buff}, nil + return &rawLogWriter{file: file, buff: buff} } // Write bytes to file @@ -141,3 +144,7 @@ func (l *rawLogWriter) Close() error { return nil } + +func openFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) +} diff --git a/server/logging/filesystem_test.go b/server/logging/filesystem_test.go index ae6328ec44..912e3e5542 100644 --- a/server/logging/filesystem_test.go +++ b/server/logging/filesystem_test.go @@ -4,6 +4,8 @@ import ( "context" "crypto/rand" "encoding/json" + stderrors "errors" + "io/fs" "io/ioutil" "os" "path" @@ -59,6 +61,27 @@ func TestFilesystemLogger(t *testing.T) { } +// TestFilesystemLoggerPermission tests that NewFilesystemLogWriter fails +// if the process does not have permissions to write to the provided path. +func TestFilesystemLoggerPermission(t *testing.T) { + tempPath := t.TempDir() + require.NoError(t, os.Chmod(tempPath, 0000)) + fileName := path.Join(tempPath, "filesystemLogWriter") + for _, tc := range []struct { + name string + rotation bool + }{ + {name: "with-rotation", rotation: true}, + {name: "without-rotation", rotation: false}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := NewFilesystemLogWriter(fileName, log.NewNopLogger(), tc.rotation, false) + require.Error(t, err) + require.True(t, stderrors.Is(err, fs.ErrPermission), err) + }) + } +} + func BenchmarkFilesystemLogger(b *testing.B) { ctx := context.Background() tempPath, err := ioutil.TempDir("", "test") From c69937945a701580d9f4dbf5acfcb728a72233bc Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 20 Sep 2021 14:47:06 -0300 Subject: [PATCH 26/82] Introduce `entityName` type for mysql entity table names (#2139) * Introduce entity type to specify mysql table names for deleteEntit* functions * Remove changes entry for issue (non-user facing changes) --- server/datastore/mysql/delete.go | 18 +++++++++--------- server/datastore/mysql/delete_test.go | 6 +++--- server/datastore/mysql/hosts.go | 2 +- server/datastore/mysql/invites.go | 2 +- server/datastore/mysql/labels.go | 2 +- server/datastore/mysql/mysql.go | 16 ++++++++++++++++ server/datastore/mysql/packs.go | 2 +- server/datastore/mysql/queries.go | 4 ++-- server/datastore/mysql/scheduled_queries.go | 2 +- server/datastore/mysql/sessions.go | 2 +- server/datastore/mysql/teams.go | 2 +- server/datastore/mysql/users.go | 2 +- 12 files changed, 38 insertions(+), 22 deletions(-) diff --git a/server/datastore/mysql/delete.go b/server/datastore/mysql/delete.go index 384327b8b0..f310769469 100644 --- a/server/datastore/mysql/delete.go +++ b/server/datastore/mysql/delete.go @@ -10,41 +10,41 @@ import ( // deleteEntity deletes an entity with the given id from the given DB table, // returning a notFound error if appropriate. -func (d *Datastore) deleteEntity(ctx context.Context, dbTable string, id uint) error { - deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, dbTable) +func (d *Datastore) deleteEntity(ctx context.Context, dbTable entity, id uint) error { + deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, dbTable.name) result, err := d.writer.ExecContext(ctx, deleteStmt, id) if err != nil { return errors.Wrapf(err, "delete %s", dbTable) } rows, _ := result.RowsAffected() if rows != 1 { - return notFound(dbTable).WithID(id) + return notFound(dbTable.name).WithID(id) } return nil } // deleteEntityByName deletes an entity with the given name from the given DB // table, returning a notFound error if appropriate. -func (d *Datastore) deleteEntityByName(ctx context.Context, dbTable string, name string) error { - deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable) +func (d *Datastore) deleteEntityByName(ctx context.Context, dbTable entity, name string) error { + deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable.name) result, err := d.writer.ExecContext(ctx, deleteStmt, name) if err != nil { if isMySQLForeignKey(err) { - return foreignKey(dbTable, name) + return foreignKey(dbTable.name, name) } return errors.Wrapf(err, "delete %s", dbTable) } rows, _ := result.RowsAffected() if rows != 1 { - return notFound(dbTable).WithName(name) + return notFound(dbTable.name).WithName(name) } return nil } // deleteEntities deletes the existing entity objects with the provided IDs. // The number of deleted entities is returned along with any error. -func (d *Datastore) deleteEntities(ctx context.Context, dbTable string, ids []uint) (uint, error) { - deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id IN (?)`, dbTable) +func (d *Datastore) deleteEntities(ctx context.Context, dbTable entity, ids []uint) (uint, error) { + deleteStmt := fmt.Sprintf(`DELETE FROM %s WHERE id IN (?)`, dbTable.name) query, args, err := sqlx.In(deleteStmt, ids) if err != nil { diff --git a/server/datastore/mysql/delete_test.go b/server/datastore/mysql/delete_test.go index abe0cc26fd..47cddd036a 100644 --- a/server/datastore/mysql/delete_test.go +++ b/server/datastore/mysql/delete_test.go @@ -29,7 +29,7 @@ func TestDeleteEntity(t *testing.T) { require.NoError(t, err) require.NotNil(t, host) - require.NoError(t, ds.deleteEntity(context.Background(), "hosts", host.ID)) + require.NoError(t, ds.deleteEntity(context.Background(), hostsTable, host.ID)) host, err = ds.Host(context.Background(), host.ID) require.Error(t, err) @@ -42,7 +42,7 @@ func TestDeleteEntityByName(t *testing.T) { query1 := test.NewQuery(t, ds, t.Name()+"time", "select * from time", 0, true) - require.NoError(t, ds.deleteEntityByName(context.Background(), "queries", query1.Name)) + require.NoError(t, ds.deleteEntityByName(context.Background(), queriesTable, query1.Name)) gotQ, err := ds.Query(context.Background(), query1.ID) require.Error(t, err) @@ -57,7 +57,7 @@ func TestDeleteEntities(t *testing.T) { query2 := test.NewQuery(t, ds, t.Name()+"time2", "select * from time", 0, true) query3 := test.NewQuery(t, ds, t.Name()+"time3", "select * from time", 0, true) - count, err := ds.deleteEntities(context.Background(), "queries", []uint{query1.ID, query2.ID}) + count, err := ds.deleteEntities(context.Background(), queriesTable, []uint{query1.ID, query2.ID}) require.NoError(t, err) assert.Equal(t, uint(2), count) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index e23566b0d6..2af56b0e2d 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -292,7 +292,7 @@ func loadHostUsersDB(ctx context.Context, db sqlx.QueryerContext, host *fleet.Ho } func (d *Datastore) DeleteHost(ctx context.Context, hid uint) error { - err := d.deleteEntity(ctx, "hosts", hid) + err := d.deleteEntity(ctx, hostsTable, hid) if err != nil { return errors.Wrapf(err, "deleting host with id %d", hid) } diff --git a/server/datastore/mysql/invites.go b/server/datastore/mysql/invites.go index bd92bb8aaa..c2d9371c75 100644 --- a/server/datastore/mysql/invites.go +++ b/server/datastore/mysql/invites.go @@ -138,7 +138,7 @@ func (d *Datastore) InviteByToken(ctx context.Context, token string) (*fleet.Inv } func (d *Datastore) DeleteInvite(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, "invites", id) + return d.deleteEntity(ctx, invitesTable, id) } func (d *Datastore) loadTeamsForInvites(ctx context.Context, invites []*fleet.Invite) error { diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 5da41d18da..a006ffe193 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -232,7 +232,7 @@ func (d *Datastore) SaveLabel(ctx context.Context, label *fleet.Label) (*fleet.L // DeleteLabel deletes a fleet.Label func (d *Datastore) DeleteLabel(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, "labels", name) + return d.deleteEntityByName(ctx, labelsTable, name) } // Label returns a fleet.Label identified by lid if one exists. diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 6d72fab858..9664df5111 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -62,6 +62,22 @@ type Datastore struct { type txFn func(sqlx.ExtContext) error +type entity struct { + name string +} + +var ( + hostsTable = entity{"hosts"} + invitesTable = entity{"invites"} + labelsTable = entity{"labels"} + packsTable = entity{"packs"} + queriesTable = entity{"queries"} + scheduledQueriesTable = entity{"scheduled_queries"} + sessionsTable = entity{"sessions"} + teamsTable = entity{"teams"} + usersTable = entity{"users"} +) + // retryableError determines whether a MySQL error can be retried. By default // errors are considered non-retryable. Only errors that we know have a // possibility of succeeding on a retry should return true in this function. diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index 20b2e89fed..3e03afc873 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -365,7 +365,7 @@ func (d *Datastore) SavePack(ctx context.Context, pack *fleet.Pack) error { // DeletePack deletes a fleet.Pack so that it won't show up in results. func (d *Datastore) DeletePack(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, "packs", name) + return d.deleteEntityByName(ctx, packsTable, name) } // Pack fetch fleet.Pack with matching ID diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index e15248ce9d..8a6eabeafd 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -138,13 +138,13 @@ func (d *Datastore) SaveQuery(ctx context.Context, q *fleet.Query) error { // DeleteQuery deletes Query identified by Query.ID. func (d *Datastore) DeleteQuery(ctx context.Context, name string) error { - return d.deleteEntityByName(ctx, "queries", name) + return d.deleteEntityByName(ctx, queriesTable, name) } // DeleteQueries deletes the existing query objects with the provided IDs. The // number of deleted queries is returned along with any error. func (d *Datastore) DeleteQueries(ctx context.Context, ids []uint) (uint, error) { - return d.deleteEntities(ctx, "queries", ids) + return d.deleteEntities(ctx, queriesTable, ids) } // Query returns a single Query identified by id, if such exists. diff --git a/server/datastore/mysql/scheduled_queries.go b/server/datastore/mysql/scheduled_queries.go index 8605f60b35..560ce5bec9 100644 --- a/server/datastore/mysql/scheduled_queries.go +++ b/server/datastore/mysql/scheduled_queries.go @@ -123,7 +123,7 @@ func saveScheduledQueryDB(ctx context.Context, exec sqlx.ExecerContext, sq *flee } func (d *Datastore) DeleteScheduledQuery(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, "scheduled_queries", id) + return d.deleteEntity(ctx, scheduledQueriesTable, id) } func (d *Datastore) ScheduledQuery(ctx context.Context, id uint) (*fleet.ScheduledQuery, error) { diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go index 16fb65c8fb..02e8273beb 100644 --- a/server/datastore/mysql/sessions.go +++ b/server/datastore/mysql/sessions.go @@ -70,7 +70,7 @@ func (d *Datastore) NewSession(ctx context.Context, session *fleet.Session) (*fl } func (d *Datastore) DestroySession(ctx context.Context, session *fleet.Session) error { - err := d.deleteEntity(ctx, "sessions", session.ID) + err := d.deleteEntity(ctx, sessionsTable, session.ID) if err != nil { return errors.Wrap(err, "deleting session") } diff --git a/server/datastore/mysql/teams.go b/server/datastore/mysql/teams.go index 122a734cc5..183f6c4dac 100644 --- a/server/datastore/mysql/teams.go +++ b/server/datastore/mysql/teams.go @@ -80,7 +80,7 @@ func saveTeamSecretsDB(ctx context.Context, exec sqlx.ExecerContext, team *fleet } func (d *Datastore) DeleteTeam(ctx context.Context, tid uint) error { - if err := d.deleteEntity(ctx, "teams", tid); err != nil { + if err := d.deleteEntity(ctx, teamsTable, tid); err != nil { return errors.Wrapf(err, "delete team id %d", tid) } return nil diff --git a/server/datastore/mysql/users.go b/server/datastore/mysql/users.go index 29399b1114..4f801295ca 100644 --- a/server/datastore/mysql/users.go +++ b/server/datastore/mysql/users.go @@ -267,5 +267,5 @@ func saveTeamsForUserDB(ctx context.Context, tx sqlx.ExtContext, user *fleet.Use // DeleteUser deletes the associated user func (d *Datastore) DeleteUser(ctx context.Context, id uint) error { - return d.deleteEntity(ctx, "users", id) + return d.deleteEntity(ctx, usersTable, id) } From 86dce785ae6339476f3673d1ba3664249b57dd6a Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 20 Sep 2021 14:09:38 -0400 Subject: [PATCH 27/82] Improve performance of the Go test suite (#2060) Closes #1805 --- .github/workflows/test-go.yaml | 2 +- cmd/fleetctl/convert_test.go | 2 - cmd/fleetctl/package_test.go | 6 +- .../vulnerability_data_stream_test.go | 4 + docs/3-Contributing/2-Testing.md | 12 +- server/datastore/mysql/activities_test.go | 32 ++- server/datastore/mysql/app_configs_test.go | 53 ++-- server/datastore/mysql/campaigns_test.go | 52 ++-- server/datastore/mysql/carves_test.go | 43 ++-- server/datastore/mysql/delete_test.go | 31 ++- server/datastore/mysql/email_changes_test.go | 19 +- server/datastore/mysql/hosts_test.go | 228 ++++++++---------- server/datastore/mysql/invites_test.go | 57 +++-- server/datastore/mysql/labels_test.go | 97 ++++---- server/datastore/mysql/packs_test.go | 109 ++++----- server/datastore/mysql/password_reset_test.go | 24 +- server/datastore/mysql/policies_test.go | 24 +- server/datastore/mysql/queries_test.go | 61 +++-- .../datastore/mysql/scheduled_queries_test.go | 48 ++-- server/datastore/mysql/sessions_test.go | 18 +- server/datastore/mysql/software.go | 13 +- server/datastore/mysql/software_test.go | 91 ++++--- server/datastore/mysql/statistics_test.go | 18 +- server/datastore/mysql/targets_test.go | 37 +-- server/datastore/mysql/teams_test.go | 44 ++-- server/datastore/mysql/testing_utils.go | 65 ++++- server/datastore/mysql/users_test.go | 54 +++-- server/vulnerabilities/cpe_test.go | 8 + server/vulnerabilities/cve_test.go | 4 + 29 files changed, 729 insertions(+), 527 deletions(-) diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 6488fc6dc1..d9a1e3d3fe 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -41,7 +41,7 @@ jobs: - name: Run Go Tests run: | - MYSQL_TEST=1 make test-go + NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 make test-go - name: Upload to Codecov uses: codecov/codecov-action@v2 diff --git a/cmd/fleetctl/convert_test.go b/cmd/fleetctl/convert_test.go index 1d0724494d..3b838aae1e 100644 --- a/cmd/fleetctl/convert_test.go +++ b/cmd/fleetctl/convert_test.go @@ -11,7 +11,6 @@ import ( ) func TestConvertFileOutput(t *testing.T) { - t.Parallel() // setup the cli and the convert command app := cli.NewApp() app.Commands = []*cli.Command{convertCommand()} @@ -41,7 +40,6 @@ func TestConvertFileOutput(t *testing.T) { } func TestConvertFileStdout(t *testing.T) { - t.Parallel() r, w, _ := os.Pipe() os.Stdout = w diff --git a/cmd/fleetctl/package_test.go b/cmd/fleetctl/package_test.go index 72b3db3561..aa4a9c468b 100644 --- a/cmd/fleetctl/package_test.go +++ b/cmd/fleetctl/package_test.go @@ -1,12 +1,16 @@ package main import ( - "github.com/stretchr/testify/require" "os" "testing" + + "github.com/stretchr/testify/require" ) func TestPackage(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } // --type is required runAppCheckErr(t, []string{"package", "deb"}, "Required flag \"type\" not set") diff --git a/cmd/fleetctl/vulnerability_data_stream_test.go b/cmd/fleetctl/vulnerability_data_stream_test.go index d3c7469eec..8fb865447a 100644 --- a/cmd/fleetctl/vulnerability_data_stream_test.go +++ b/cmd/fleetctl/vulnerability_data_stream_test.go @@ -12,6 +12,10 @@ import ( ) func TestVulnerabilityDataStream(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + runAppCheckErr(t, []string{"vulnerability-data-stream"}, "No directory provided") vulnPath := t.TempDir() diff --git a/docs/3-Contributing/2-Testing.md b/docs/3-Contributing/2-Testing.md index 73ef0a7d42..6fc7e5f756 100644 --- a/docs/3-Contributing/2-Testing.md +++ b/docs/3-Contributing/2-Testing.md @@ -40,7 +40,7 @@ go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.0 Make sure it is available in your PATH. To execute the basic unit and integration tests, run the following from the root of the repository: ``` -MYSQL_TEST=1 make test +REDIS_TEST=1 MYSQL_TEST=1 make test ``` It is a good idea to run `make test` before submitting a Pull Request. @@ -50,7 +50,7 @@ It is a good idea to run `make test` before submitting a Pull Request. To run all Go unit tests, run the following: ``` -make test-go +REDIS_TEST=1 MYSQL_TEST=1 make test-go ``` #### Go linters @@ -105,6 +105,14 @@ To run email related integration tests using MailHog set environment as follows: MAIL_TEST=1 make test-go ``` +#### Network tests + +A few tests require network access as they make requests to external hosts. Given that the network is unreliable, may not be available, and those hosts may also not be unavailable, those tests are skipped by default and are opt-in via the `NETWORK_TEST` environment variable. To run them: + +``` +NETWORK_TEST=1 make test-go +``` + ### Viewing test coverage When you run `make test` or `make test-go` from the root of the repository, test coverage reports are generated in every subpackage. For example, the `server` subpackage will have a coverage report generated in `./server/server.cover` diff --git a/server/datastore/mysql/activities_test.go b/server/datastore/mysql/activities_test.go index 5dbdc7b6fc..af97ca730f 100644 --- a/server/datastore/mysql/activities_test.go +++ b/server/datastore/mysql/activities_test.go @@ -10,10 +10,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestActivityUsernameChange(t *testing.T) { +func TestActivity(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"UsernameChange", testActivityUsernameChange}, + {"New", testActivityNew}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testActivityUsernameChange(t *testing.T, ds *Datastore) { u := &fleet.User{ Password: []byte("asd"), Name: "fullname", @@ -52,10 +67,7 @@ func TestActivityUsernameChange(t *testing.T) { assert.Nil(t, activities[0].ActorGravatar) } -func TestNewActivity(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testActivityNew(t *testing.T, ds *Datastore) { u := &fleet.User{ Password: []byte("asd"), Name: "fullname", @@ -86,4 +98,12 @@ func TestNewActivity(t *testing.T) { assert.Len(t, activities, 1) assert.Equal(t, "fullname", activities[0].ActorFullName) assert.Equal(t, "test2", activities[0].Type) + + opt = fleet.ListOptions{ + Page: 0, + PerPage: 10, + } + activities, err = ds.ListActivities(context.Background(), opt) + require.NoError(t, err) + assert.Len(t, activities, 2) } diff --git a/server/datastore/mysql/app_configs_test.go b/server/datastore/mysql/app_configs_test.go index 58b0e3906d..8959912dad 100644 --- a/server/datastore/mysql/app_configs_test.go +++ b/server/datastore/mysql/app_configs_test.go @@ -14,10 +14,29 @@ import ( "github.com/stretchr/testify/require" ) -func TestOrgInfo(t *testing.T) { +func TestAppConfig(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"OrgInfo", testAppConfigOrgInfo}, + {"AdditionalQueries", testAppConfigAdditionalQueries}, + {"EnrollSecrets", testAppConfigEnrollSecrets}, + {"EnrollSecretsCaseSensitive", testAppConfigEnrollSecretsCaseSensitive}, + {"EnrollSecretRoundtrip", testAppConfigEnrollSecretRoundtrip}, + {"EnrollSecretUniqueness", testAppConfigEnrollSecretUniqueness}, + {"Defaults", testAppConfigDefaults}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.fn(t, ds) + }) + } +} + +func testAppConfigOrgInfo(t *testing.T, ds *Datastore) { info := &fleet.AppConfig{ OrgInfo: fleet.OrgInfo{ OrgName: "Test", @@ -94,10 +113,7 @@ func TestOrgInfo(t *testing.T) { assert.False(t, verify.SSOEnabled) } -func TestAdditionalQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testAppConfigAdditionalQueries(t *testing.T, ds *Datastore) { additional := ptr.RawMessage(json.RawMessage("not valid json")) info := &fleet.AppConfig{ OrgInfo: fleet.OrgInfo{ @@ -123,9 +139,8 @@ func TestAdditionalQueries(t *testing.T) { assert.JSONEq(t, `{"foo":"bar"}`, string(rawJson)) } -func TestEnrollSecrets(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecrets(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -179,9 +194,8 @@ func TestEnrollSecrets(t *testing.T) { assert.Equal(t, (*uint)(nil), secret.TeamID) } -func TestEnrollSecretsCaseSensitive(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecretsCaseSensitive(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) err := ds.ApplyEnrollSecrets( context.Background(), @@ -198,9 +212,8 @@ func TestEnrollSecretsCaseSensitive(t *testing.T) { assert.Error(t, err, "enroll secret with different case should not verify") } -func TestEnrollSecretRoundtrip(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecretRoundtrip(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -240,9 +253,8 @@ func TestEnrollSecretRoundtrip(t *testing.T) { } -func TestEnrollSecretUniqueness(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testAppConfigEnrollSecretUniqueness(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -258,10 +270,7 @@ func TestEnrollSecretUniqueness(t *testing.T) { require.Error(t, err) } -func TestAppConfigDefaults(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testAppConfigDefaults(t *testing.T, ds *Datastore) { insertAppConfigQuery := `INSERT INTO app_config_json(json_value) VALUES(?) ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)` _, err := ds.writer.Exec(insertAppConfigQuery, `{}`) require.NoError(t, err) diff --git a/server/datastore/mysql/campaigns_test.go b/server/datastore/mysql/campaigns_test.go index 245795981f..87d9316b6a 100644 --- a/server/datastore/mysql/campaigns_test.go +++ b/server/datastore/mysql/campaigns_test.go @@ -12,24 +12,30 @@ import ( "github.com/stretchr/testify/require" ) -func checkTargets(t *testing.T, ds fleet.Datastore, campaignID uint, expectedTargets fleet.HostTargets) { - targets, err := ds.DistributedQueryCampaignTargetIDs(context.Background(), campaignID) - require.Nil(t, err) - assert.ElementsMatch(t, expectedTargets.HostIDs, targets.HostIDs) - assert.ElementsMatch(t, expectedTargets.LabelIDs, targets.LabelIDs) - assert.ElementsMatch(t, expectedTargets.TeamIDs, targets.TeamIDs) +func TestCampaigns(t *testing.T) { + ds := CreateMySQLDS(t) + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"DistributedQuery", testCampaignsDistributedQuery}, + {"CleanupDistributedQuery", testCampaignsCleanupDistributedQuery}, + {"SaveDistributedQuery", testCampaignsSaveDistributedQuery}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + + c.fn(t, ds) + }) + } } -func TestDistributedQueryCampaign(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCampaignsDistributedQuery(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) - mockClock := clock.NewMockClock() - query := test.NewQuery(t, ds, "test", "select * from time", user.ID, false) - campaign := test.NewCampaign(t, ds, query.ID, fleet.QueryRunning, mockClock.Now()) { @@ -71,13 +77,9 @@ func TestDistributedQueryCampaign(t *testing.T) { test.AddHostToCampaign(t, ds, campaign.ID, h3.ID) checkTargets(t, ds, campaign.ID, fleet.HostTargets{HostIDs: []uint{h1.ID, h2.ID, h3.ID}, LabelIDs: []uint{l1.ID, l2.ID}}) - } -func TestCleanupDistributedQueryCampaigns(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCampaignsCleanupDistributedQuery(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) mockClock := clock.NewMockClock() @@ -149,13 +151,9 @@ func TestCleanupDistributedQueryCampaigns(t *testing.T) { assert.Equal(t, c2.QueryID, retrieved.QueryID) assert.Equal(t, fleet.QueryComplete, retrieved.Status) } - } -func TestSaveDistributedQueryCampaign(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCampaignsSaveDistributedQuery(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, t.Name(), t.Name()+"zwass@fleet.co", true) mockClock := clock.NewMockClock() @@ -174,3 +172,11 @@ func TestSaveDistributedQueryCampaign(t *testing.T) { require.NoError(t, err) require.Equal(t, fleet.QueryComplete, gotC.Status) } + +func checkTargets(t *testing.T, ds fleet.Datastore, campaignID uint, expectedTargets fleet.HostTargets) { + targets, err := ds.DistributedQueryCampaignTargetIDs(context.Background(), campaignID) + require.Nil(t, err) + assert.ElementsMatch(t, expectedTargets.HostIDs, targets.HostIDs) + assert.ElementsMatch(t, expectedTargets.LabelIDs, targets.LabelIDs) + assert.ElementsMatch(t, expectedTargets.TeamIDs, targets.TeamIDs) +} diff --git a/server/datastore/mysql/carves_test.go b/server/datastore/mysql/carves_test.go index 22f84f772d..2b50ee4298 100644 --- a/server/datastore/mysql/carves_test.go +++ b/server/datastore/mysql/carves_test.go @@ -14,10 +14,28 @@ import ( var mockCreatedAt = time.Now().UTC().Truncate(time.Second) -func TestCarveMetadata(t *testing.T) { +func TestCarves(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Metadata", testCarvesMetadata}, + {"Blocks", testCarvesBlocks}, + {"Cleanup", testCarvesCleanup}, + {"List", testCarvesList}, + {"Update", testCarvesUpdate}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testCarvesMetadata(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) expectedCarve := &fleet.CarveMetadata{ @@ -75,10 +93,7 @@ func TestCarveMetadata(t *testing.T) { assert.Equal(t, expectedCarve, carve) } -func TestCarveBlocks(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesBlocks(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) blockCount := int64(25) @@ -116,13 +131,9 @@ func TestCarveBlocks(t *testing.T) { require.NoError(t, err, "get block %d %v", i, expectedBlocks[i]) assert.Equal(t, expectedBlocks[i], data) } - } -func TestCarveCleanupCarves(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesCleanup(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) blockCount := int64(25) @@ -174,10 +185,7 @@ func TestCarveCleanupCarves(t *testing.T) { assert.True(t, carve.Expired) } -func TestCarveListCarves(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesList(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) expectedCarve := &fleet.CarveMetadata{ @@ -235,10 +243,7 @@ func TestCarveListCarves(t *testing.T) { assert.Len(t, carves, 2) } -func TestCarveUpdateCarve(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testCarvesUpdate(t *testing.T, ds *Datastore) { h := test.NewHost(t, ds, "foo.local", "192.168.1.10", "1", "1", time.Now()) actualCount := int64(10) diff --git a/server/datastore/mysql/delete_test.go b/server/datastore/mysql/delete_test.go index 47cddd036a..e193b2b8c5 100644 --- a/server/datastore/mysql/delete_test.go +++ b/server/datastore/mysql/delete_test.go @@ -11,9 +11,26 @@ import ( "github.com/stretchr/testify/require" ) -func TestDeleteEntity(t *testing.T) { +func TestDelete(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Entity", testDeleteEntity}, + {"EntityByName", testDeleteEntityByName}, + {"Entities", testDeleteEntities}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.fn(t, ds) + }) + } +} + +func testDeleteEntity(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -36,9 +53,8 @@ func TestDeleteEntity(t *testing.T) { assert.Nil(t, host) } -func TestDeleteEntityByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testDeleteEntityByName(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) query1 := test.NewQuery(t, ds, t.Name()+"time", "select * from time", 0, true) @@ -49,9 +65,8 @@ func TestDeleteEntityByName(t *testing.T) { assert.Nil(t, gotQ) } -func TestDeleteEntities(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() +func testDeleteEntities(t *testing.T, ds *Datastore) { + defer TruncateTables(t, ds) query1 := test.NewQuery(t, ds, t.Name()+"time1", "select * from time", 0, true) query2 := test.NewQuery(t, ds, t.Name()+"time2", "select * from time", 0, true) diff --git a/server/datastore/mysql/email_changes_test.go b/server/datastore/mysql/email_changes_test.go index e60a1d2a3f..1d8a84d5e1 100644 --- a/server/datastore/mysql/email_changes_test.go +++ b/server/datastore/mysql/email_changes_test.go @@ -11,10 +11,24 @@ import ( "github.com/stretchr/testify/require" ) -func TestChangeEmail(t *testing.T) { +func TestEmailChanges(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Confirm", testEmailChangesConfirm}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testEmailChangesConfirm(t *testing.T, ds *Datastore) { user := &fleet.User{ Password: []byte("foobar"), Email: "bob@bob.com", @@ -45,5 +59,4 @@ func TestChangeEmail(t *testing.T) { require.Nil(t, err) _, err = ds.ConfirmPendingEmailChange(context.Background(), otheruser.ID, "uniquetoken") assert.NotNil(t, err) - } diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 323cec3d7b..222ca90b06 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -45,10 +46,52 @@ var enrollTests = []struct { }, } -func TestSaveHosts(t *testing.T) { +func TestHosts(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Save", testHostsSave}, + {"DeleteWithSoftware", testHostsDeleteWithSoftware}, + {"SavePackStats", testHostsSavePackStats}, + {"SavePackStatsOverwrites", testHostsSavePackStatsOverwrites}, + {"IgnoresTeamPackStats", testHostsIgnoresTeamPackStats}, + {"Delete", testHostsDelete}, + {"ListFilterAdditional", testHostsListFilterAdditional}, + {"ListStatus", testHostsListStatus}, + {"ListQuery", testHostsListQuery}, + {"Enroll", testHostsEnroll}, + {"Authenticate", testHostsAuthenticate}, + {"AuthenticateCaseSensitive", testHostsAuthenticateCaseSensitive}, + {"Search", testHostsSearch}, + {"SearchLimit", testHostsSearchLimit}, + {"GenerateStatusStatistics", testHostsGenerateStatusStatistics}, + {"MarkSeen", testHostsMarkSeen}, + {"MarkSeenMany", testHostsMarkSeenMany}, + {"CleanupIncoming", testHostsCleanupIncoming}, + {"IDsByName", testHostsIDsByName}, + {"Additional", testHostsAdditional}, + {"ByIdentifier", testHostsByIdentifier}, + {"AddToTeam", testHostsAddToTeam}, + {"SaveUsers", testHostsSaveUsers}, + {"SaveUsersWithoutUid", testHostsSaveUsersWithoutUid}, + {"TotalAndUnseenSince", testHostsTotalAndUnseenSince}, + {"ListByPolicy", testHostsListByPolicy}, + {"SaveTonsOfUsers", testHostsSaveTonsOfUsers}, + {"SavePackStatsConcurrent", testHostsSavePackStatsConcurrent}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + + c.fn(t, ds) + }) + } +} + +func testHostsSave(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -99,10 +142,7 @@ func TestSaveHosts(t *testing.T) { assert.Nil(t, host) } -func TestDeleteHostWithSoftware(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsDeleteWithSoftware(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -135,10 +175,7 @@ func TestDeleteHostWithSoftware(t *testing.T) { assert.Nil(t, host) } -func TestSaveHostPackStats(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSavePackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -250,10 +287,7 @@ func TestSaveHostPackStats(t *testing.T) { require.Len(t, host.PackStats, 2) } -func TestSaveHostPackStatsOverwrites(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSavePackStatsOverwrites(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -396,10 +430,7 @@ func TestSaveHostPackStatsOverwrites(t *testing.T) { assert.Equal(t, execTime2, gotHost.PackStats[0].QueryStats[0].LastExecuted) } -func TestIgnoresTeamPackStats(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsIgnoresTeamPackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -478,10 +509,7 @@ func TestIgnoresTeamPackStats(t *testing.T) { assert.ElementsMatch(t, host.PackStats[0].QueryStats, stats1) } -func TestDeleteHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsDelete(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -500,10 +528,7 @@ func TestDeleteHost(t *testing.T) { assert.NotNil(t, err) } -func TestListHostsFilterAdditional(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListFilterAdditional(t *testing.T, ds *Datastore) { h, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -540,10 +565,7 @@ func TestListHostsFilterAdditional(t *testing.T) { assert.Equal(t, &additional, hosts[0].Additional) } -func TestListHostsStatus(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListStatus(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -579,10 +601,7 @@ func TestListHostsStatus(t *testing.T) { assert.Equal(t, 10, len(hosts)) } -func TestListHostsQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListQuery(t *testing.T, ds *Datastore) { hosts := []*fleet.Host{} for i := 0; i < 10; i++ { host, err := ds.NewHost(context.Background(), &fleet.Host{ @@ -669,10 +688,7 @@ func TestListHostsQuery(t *testing.T) { assert.Equal(t, 1, len(gotHosts)) } -func TestEnrollHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsEnroll(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) team, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) @@ -709,10 +725,7 @@ func TestEnrollHost(t *testing.T) { } } -func TestAuthenticateHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAuthenticate(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) for _, tt := range enrollTests { h, err := ds.EnrollHost(context.Background(), tt.uuid, tt.nodeKey, nil, 0) @@ -730,10 +743,7 @@ func TestAuthenticateHost(t *testing.T) { assert.Error(t, err) } -func TestAuthenticateHostCaseSensitive(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAuthenticateCaseSensitive(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) for _, tt := range enrollTests { h, err := ds.EnrollHost(context.Background(), tt.uuid, tt.nodeKey, nil, 0) @@ -744,10 +754,7 @@ func TestAuthenticateHostCaseSensitive(t *testing.T) { } } -func TestSearchHosts(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSearch(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), @@ -836,10 +843,7 @@ func TestSearchHosts(t *testing.T) { assert.Equal(t, 1, len(hits)) } -func TestSearchHostsLimit(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSearchLimit(t *testing.T, ds *Datastore) { filter := fleet.TeamFilter{User: test.UserAdmin} for i := 0; i < 15; i++ { @@ -860,10 +864,7 @@ func TestSearchHostsLimit(t *testing.T) { assert.Len(t, hosts, 10) } -func TestGenerateHostStatusStatistics(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { filter := fleet.TeamFilter{User: test.UserAdmin} mockClock := clock.NewMockClock() @@ -942,10 +943,7 @@ func TestGenerateHostStatusStatistics(t *testing.T) { assert.Equal(t, uint(4), new) } -func TestMarkHostSeen(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsMarkSeen(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() anHourAgo := mockClock.Now().Add(-1 * time.Hour).UTC() @@ -980,10 +978,7 @@ func TestMarkHostSeen(t *testing.T) { } } -func TestMarkHostsSeen(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() aSecondAgo := mockClock.Now().Add(-1 * time.Second).UTC() @@ -1041,13 +1036,9 @@ func TestMarkHostsSeen(t *testing.T) { require.NotNil(t, h2Verify) assert.WithinDuration(t, aSecondAgo, h2Verify.SeenTime, time.Second) } - } -func TestCleanupIncomingHosts(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() h1, err := ds.NewHost(context.Background(), &fleet.Host{ @@ -1093,10 +1084,7 @@ func TestCleanupIncomingHosts(t *testing.T) { assert.Nil(t, err) } -func TestHostIDsByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsIDsByName(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -1117,10 +1105,7 @@ func TestHostIDsByName(t *testing.T) { assert.Equal(t, hosts, []uint{2, 3, 6}) } -func TestHostAdditional(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAdditional(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -1191,10 +1176,7 @@ func TestHostAdditional(t *testing.T) { assert.Equal(t, &additional, h.Additional) } -func TestHostByIdentifier(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsByIdentifier(t *testing.T, ds *Datastore) { for i := 1; i <= 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -1232,10 +1214,7 @@ func TestHostByIdentifier(t *testing.T) { require.Error(t, err) } -func TestAddHostsToTeam(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsAddToTeam(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) @@ -1282,10 +1261,7 @@ func TestAddHostsToTeam(t *testing.T) { } } -func TestSaveHostUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSaveUsers(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -1354,10 +1330,7 @@ func TestSaveHostUsers(t *testing.T) { test.ElementsMatchSkipID(t, host.Users, []fleet.HostUser{u1, u2}) } -func TestSaveUsersWithoutUid(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSaveUsersWithoutUid(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -1428,10 +1401,7 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { require.NotNil(t, host) } -func TestTotalAndUnseenHostsSince(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { addHostSeenLast(t, ds, 1, 0) total, unseen, err := ds.TotalAndUnseenHostsSince(context.Background(), 1) @@ -1448,10 +1418,7 @@ func TestTotalAndUnseenHostsSince(t *testing.T) { assert.Equal(t, 2, unseen) } -func TestListHostsByPolicy(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsListByPolicy(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), @@ -1506,10 +1473,7 @@ func TestListHostsByPolicy(t *testing.T) { require.Len(t, hosts, 8) } -func TestSaveTonsOfUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -1545,7 +1509,12 @@ func TestSaveTonsOfUsers(t *testing.T) { var count1 int32 var count2 int32 + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for { host1, err := ds.Host(context.Background(), host1.ID) if err != nil { @@ -1584,7 +1553,9 @@ func TestSaveTonsOfUsers(t *testing.T) { errCh <- err return } - atomic.AddInt32(&count1, 1) + if atomic.AddInt32(&count1, 1) >= 100 { + return + } select { case <-ctx.Done(): @@ -1595,6 +1566,8 @@ func TestSaveTonsOfUsers(t *testing.T) { }() go func() { + defer wg.Done() + for { host2, err := ds.Host(context.Background(), host2.ID) if err != nil { @@ -1633,7 +1606,9 @@ func TestSaveTonsOfUsers(t *testing.T) { errCh <- err return } - atomic.AddInt32(&count2, 1) + if atomic.AddInt32(&count2, 1) >= 100 { + return + } select { case <-ctx.Done(): @@ -1644,21 +1619,24 @@ func TestSaveTonsOfUsers(t *testing.T) { }() ticker := time.NewTicker(10 * time.Second) + go func() { + wg.Wait() + cancelFunc() + }() select { case err := <-errCh: - require.NoError(t, err) cancelFunc() + require.NoError(t, err) + case <-ctx.Done(): case <-ticker.C: + require.Fail(t, "timed out") } - fmt.Println("Count1", atomic.LoadInt32(&count1)) - fmt.Println("Count2", atomic.LoadInt32(&count2)) + t.Log("Count1", atomic.LoadInt32(&count1)) + t.Log("Count2", atomic.LoadInt32(&count2)) } -func TestSaveHostPackStatsConcurrent(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -1743,14 +1721,18 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { }, }, } - return ds.SaveHost(context.Background(), host1) + return ds.SaveHost(context.Background(), host) } errCh := make(chan error) var counter int32 - total := int32(1000) + const total = int32(100) + + var wg sync.WaitGroup loopAndSaveHost := func(host *fleet.Host) { + defer wg.Done() + for { err := saveHostRandomStats(host) if err != nil { @@ -1770,10 +1752,13 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { } } + wg.Add(3) go loopAndSaveHost(host1) go loopAndSaveHost(host2) go func() { + defer wg.Done() + for { specs := []*fleet.PackSpec{ { @@ -1817,13 +1802,14 @@ func TestSaveHostPackStatsConcurrent(t *testing.T) { } }() - ticker := time.NewTicker(1 * time.Minute) + ticker := time.NewTicker(10 * time.Second) select { case err := <-errCh: + cancelFunc() require.NoError(t, err) case <-ctx.Done(): - return + wg.Wait() case <-ticker.C: - t.Fail() + require.Fail(t, "timed out") } } diff --git a/server/datastore/mysql/invites_test.go b/server/datastore/mysql/invites_test.go index a1b28edbc5..b1f6a0e587 100644 --- a/server/datastore/mysql/invites_test.go +++ b/server/datastore/mysql/invites_test.go @@ -12,10 +12,29 @@ import ( "gopkg.in/guregu/null.v3" ) -func TestCreateInvite(t *testing.T) { +func TestInvites(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Create", testInvitesCreate}, + {"List", testInvitesList}, + {"Delete", testInvitesDelete}, + {"ByToken", testInvitesByToken}, + {"ByEmail", testInvitesByEmail}, + {"Invite", testInvitesInvite}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testInvitesCreate(t *testing.T, ds *Datastore) { for i := 0; i < 3; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -64,13 +83,9 @@ func setupTestInvites(t *testing.T, ds fleet.Datastore) { _, err := ds.NewInvite(context.Background(), &i) require.NoError(t, err, "Failure creating user", user) } - } -func TestListInvites(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesList(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) opt := fleet.ListOptions{ @@ -94,13 +109,9 @@ func TestListInvites(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 4, len(result)) // allow for admin we created assert.Equal(t, "User00", result[3].Name) - } -func TestDeleteInvite(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesDelete(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) invite, err := ds.InviteByEmail(context.Background(), "user0@foo.com") @@ -114,13 +125,9 @@ func TestDeleteInvite(t *testing.T) { invite, err = ds.InviteByEmail(context.Background(), "user0@foo.com") assert.NotNil(t, err) assert.Nil(t, invite) - } -func TestInviteByToken(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesByToken(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) var inviteTests = []struct { @@ -137,7 +144,7 @@ func TestInviteByToken(t *testing.T) { } for _, tt := range inviteTests { - t.Run("", func(t *testing.T) { + t.Run(tt.token, func(t *testing.T) { invite, err := ds.InviteByToken(context.Background(), tt.token) if tt.wantErr != nil { require.NotNil(t, err) @@ -146,15 +153,11 @@ func TestInviteByToken(t *testing.T) { } require.Nil(t, err) assert.NotEqual(t, invite.ID, 0) - }) } } -func TestInviteByEmail(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesByEmail(t *testing.T, ds *Datastore) { setupTestInvites(t, ds) var inviteTests = []struct { @@ -171,7 +174,7 @@ func TestInviteByEmail(t *testing.T) { } for _, tt := range inviteTests { - t.Run("", func(t *testing.T) { + t.Run(tt.email, func(t *testing.T) { invite, err := ds.InviteByEmail(context.Background(), tt.email) if tt.wantErr != nil { require.NotNil(t, err) @@ -180,15 +183,11 @@ func TestInviteByEmail(t *testing.T) { } require.Nil(t, err) assert.NotEqual(t, invite.ID, 0) - }) } } -func TestInvite(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testInvitesInvite(t *testing.T, ds *Datastore) { admin := &fleet.Invite{ Email: "admin@foo.com", Name: "Xadmin", diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index 30fef5d73e..dabc903216 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -37,9 +37,36 @@ func TestBatchHostnamesLarge(t *testing.T) { } func TestLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() + ds := CreateMySQLDS(t) + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"AddAllHosts", testLabelsAddAllHosts}, + {"Search", testLabelsSearch}, + {"ListHostsInLabel", testLabelsListHostsInLabel}, + {"ListHostsInLabelAndStatus", testLabelsListHostsInLabelAndStatus}, + {"ListHostsInLabelAndTeamFilter", testLabelsListHostsInLabelAndTeamFilter}, + {"BuiltIn", testLabelsBuiltIn}, + {"ListUniqueHostsInLabels", testLabelsListUniqueHostsInLabels}, + {"ChangeDetails", testLabelsChangeDetails}, + {"GetSpec", testLabelsGetSpec}, + {"ApplySpecsRoundtrip", testLabelsApplySpecsRoundtrip}, + {"IDsByName", testLabelsIDsByName}, + {"Save", testLabelsSave}, + {"QueriesForCentOSHost", testLabelsQueriesForCentOSHost}, + {"RecordNonExistentQueryLabelExecution", testLabelsRecordNonexistentQueryLabelExecution}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testLabelsAddAllHosts(t *testing.T, db *Datastore) { test.AddAllHostsLabel(t, db) hosts := []fleet.Host{} var host *fleet.Host @@ -185,10 +212,7 @@ func TestLabels(t *testing.T) { assert.Len(t, labels, 1) } -func TestSearchLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsSearch(t *testing.T, db *Datastore) { specs := []*fleet.LabelSpec{ {ID: 1, Name: "foo"}, {ID: 2, Name: "bar"}, @@ -241,10 +265,7 @@ func TestSearchLabels(t *testing.T) { assert.Contains(t, labels, all) } -func TestListHostsInLabel(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -306,10 +327,7 @@ func TestListHostsInLabel(t *testing.T) { } } -func TestListHostsInLabelAndStatus(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -362,10 +380,7 @@ func TestListHostsInLabelAndStatus(t *testing.T) { } } -func TestListHostsInLabelAndTeamFilter(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), @@ -439,10 +454,7 @@ func TestListHostsInLabelAndTeamFilter(t *testing.T) { } } -func TestBuiltInLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsBuiltIn(t *testing.T, db *Datastore) { require.Nil(t, db.MigrateData(context.Background())) user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} @@ -456,10 +468,7 @@ func TestBuiltInLabels(t *testing.T) { assert.Equal(t, fleet.LabelTypeBuiltIn, hits[1].LabelType) } -func TestListUniqueHostsInLabels(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsListUniqueHostsInLabels(t *testing.T, db *Datastore) { hosts := []*fleet.Host{} for i := 0; i < 4; i++ { h, err := db.NewHost(context.Background(), &fleet.Host{ @@ -508,13 +517,9 @@ func TestListUniqueHostsInLabels(t *testing.T) { labels, err := db.ListLabels(context.Background(), filter, fleet.ListOptions{}) require.Nil(t, err) require.Len(t, labels, 2) - } -func TestChangeLabelDetails(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsChangeDetails(t *testing.T, db *Datastore) { label := fleet.LabelSpec{ ID: 1, Name: "my label", @@ -583,10 +588,7 @@ func setupLabelSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.LabelSpec { return expectedSpecs } -func TestGetLabelSpec(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testLabelsGetSpec(t *testing.T, ds *Datastore) { expectedSpecs := setupLabelSpecsTest(t, ds) for _, s := range expectedSpecs { @@ -596,10 +598,7 @@ func TestGetLabelSpec(t *testing.T) { } } -func TestApplyLabelSpecsRoundtrip(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testLabelsApplySpecsRoundtrip(t *testing.T, ds *Datastore) { expectedSpecs := setupLabelSpecsTest(t, ds) specs, err := ds.GetLabelSpecs(context.Background()) @@ -614,10 +613,7 @@ func TestApplyLabelSpecsRoundtrip(t *testing.T) { test.ElementsMatchSkipTimestampsID(t, expectedSpecs, specs) } -func TestLabelIDsByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testLabelsIDsByName(t *testing.T, ds *Datastore) { setupLabelSpecsTest(t, ds) labels, err := ds.LabelIDsByName(context.Background(), []string{"foo", "bar", "bing"}) @@ -626,10 +622,7 @@ func TestLabelIDsByName(t *testing.T) { assert.Equal(t, []uint{1, 2, 3}, labels) } -func TestSaveLabel(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsSave(t *testing.T, db *Datastore) { label := &fleet.Label{ Name: "my label", Description: "a label", @@ -648,10 +641,7 @@ func TestSaveLabel(t *testing.T) { assert.Equal(t, label.Description, saved.Description) } -func TestLabelQueriesForCentOSHost(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsQueriesForCentOSHost(t *testing.T, db *Datastore) { host, err := db.EnrollHost(context.Background(), "0", "0", nil, 0) require.Nil(t, err, "enrollment should succeed") host.Platform = "rhel" @@ -680,10 +670,7 @@ func TestLabelQueriesForCentOSHost(t *testing.T) { assert.Equal(t, "select 1;", queries[fmt.Sprint(label.ID)]) } -func TestRecordNonexistentQueryLabelExecution(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() - +func testLabelsRecordNonexistentQueryLabelExecution(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 710d752a22..43d97c94d8 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -15,10 +15,39 @@ import ( "github.com/stretchr/testify/require" ) -func TestDeletePack(t *testing.T) { +func TestPacks(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Delete", testPacksDelete}, + {"Save", testPacksSave}, + {"GetByName", testPacksGetByName}, + {"List", testPacksList}, + {"ApplySpecRoundtrip", testPacksApplySpecRoundtrip}, + {"GetSpec", testPacksGetSpec}, + {"ApplySpecMissingQueries", testPacksApplySpecMissingQueries}, + {"ApplySpecMissingName", testPacksApplySpecMissingName}, + {"ListForHost", testPacksListForHost}, + {"EnsureGlobal", testPacksEnsureGlobal}, + {"EnsureTeam", testPacksEnsureTeam}, + {"TeamNameChangesTeamSchedule", testPacksTeamNameChangesTeamSchedule}, + {"TeamScheduleNamesMigrateToNewFormat", testPacksTeamScheduleNamesMigrateToNewFormat}, + {"ApplySpecFailsOnTargetIDNull", testPacksApplySpecFailsOnTargetIDNull}, + {"ApplyStatsNotLocking", testPacksApplyStatsNotLocking}, + {"ApplyStatsNotLockingTryTwo", testPacksApplyStatsNotLockingTryTwo}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testPacksDelete(t *testing.T, ds *Datastore) { pack := test.NewPack(t, ds, "foo") assert.NotEqual(t, uint(0), pack.ID) @@ -33,10 +62,7 @@ func TestDeletePack(t *testing.T) { assert.NotNil(t, err) } -func TestSavePack(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksSave(t *testing.T, ds *Datastore) { expectedPack := &fleet.Pack{ Name: "foo", HostIDs: []uint{1}, @@ -70,10 +96,7 @@ func TestSavePack(t *testing.T) { test.EqualSkipTimestampsID(t, expectedPack, pack) } -func TestGetPackByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksGetByName(t *testing.T, ds *Datastore) { pack := test.NewPack(t, ds, "foo") assert.NotEqual(t, uint(0), pack.ID) @@ -87,13 +110,9 @@ func TestGetPackByName(t *testing.T) { require.Nil(t, err) assert.False(t, ok) assert.Nil(t, pack) - } -func TestListPacks(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksList(t *testing.T, ds *Datastore) { p1 := &fleet.PackSpec{ ID: 1, Name: "foo_pack", @@ -223,10 +242,7 @@ func setupPackSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.PackSpec { return expectedSpecs } -func TestApplyPackSpecRoundtrip(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecRoundtrip(t *testing.T, ds *Datastore) { expectedSpecs := setupPackSpecsTest(t, ds) gotSpec, err := ds.GetPackSpecs(context.Background()) @@ -234,10 +250,7 @@ func TestApplyPackSpecRoundtrip(t *testing.T) { assert.Equal(t, expectedSpecs, gotSpec) } -func TestGetPackSpec(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksGetSpec(t *testing.T, ds *Datastore) { expectedSpecs := setupPackSpecsTest(t, ds) for _, s := range expectedSpecs { @@ -247,10 +260,7 @@ func TestGetPackSpec(t *testing.T) { } } -func TestApplyPackSpecMissingQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecMissingQueries(t *testing.T, ds *Datastore) { // Do not define queries mentioned in spec specs := []*fleet.PackSpec{ { @@ -275,10 +285,7 @@ func TestApplyPackSpecMissingQueries(t *testing.T) { } } -func TestApplyPackSpecMissingName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecMissingName(t *testing.T, ds *Datastore) { setupPackSpecsTest(t, ds) specs := []*fleet.PackSpec{ @@ -304,10 +311,7 @@ func TestApplyPackSpecMissingName(t *testing.T) { assert.Equal(t, "foo", spec.Queries[0].Name) } -func TestListPacksForHost(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksListForHost(t *testing.T, ds *Datastore) { mockClock := clock.NewMockClock() l1 := &fleet.LabelSpec{ @@ -416,10 +420,7 @@ func TestListPacksForHost(t *testing.T) { } } -func TestEnsureGlobalPack(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksEnsureGlobal(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) packs, err := ds.ListPacks(context.Background(), fleet.PackListOptions{IncludeSystemPacks: true}) @@ -450,10 +451,7 @@ func TestEnsureGlobalPack(t *testing.T) { assert.Equal(t, "global", *gp.Type) } -func TestEnsureTeamPack(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksEnsureTeam(t *testing.T, ds *Datastore) { packs, err := ds.ListPacks(context.Background(), fleet.PackListOptions{IncludeSystemPacks: true}) require.Nil(t, err) assert.Len(t, packs, 0) @@ -499,10 +497,7 @@ func TestEnsureTeamPack(t *testing.T) { assert.Equal(t, []uint{team2.ID}, tp2.TeamIDs) } -func TestTeamNameChangesTeamSchedule(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksTeamNameChangesTeamSchedule(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -521,10 +516,7 @@ func TestTeamNameChangesTeamSchedule(t *testing.T) { assert.Equal(t, teamScheduleName(team1), tp.Name) } -func TestTeamScheduleNamesMigrateToNewFormat(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksTeamScheduleNamesMigrateToNewFormat(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) @@ -547,10 +539,7 @@ func TestTeamScheduleNamesMigrateToNewFormat(t *testing.T) { require.Equal(t, teamScheduleName(team1), tp.Name) } -func TestApplyPackSpecFailsOnTargetIDNull(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPacksApplySpecFailsOnTargetIDNull(t *testing.T, ds *Datastore) { // Do not define queries mentioned in spec specs := []*fleet.PackSpec{ { @@ -602,12 +591,9 @@ func randomPackStatsForHost(hostID, packID uint, scheduledQueries []*fleet.Sched } } -func TestPackApplyStatsNotLocking(t *testing.T) { +func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { t.Skip("This can be too much for the test db if you're running all tests") - ds := CreateMySQLDS(t) - defer ds.Close() - specs := setupPackSpecsTest(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ @@ -650,12 +636,9 @@ func TestPackApplyStatsNotLocking(t *testing.T) { cancelFunc() } -func TestPackApplyStatsNotLockingTryTwo(t *testing.T) { +func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { t.Skip("This can be too much for the test db if you're running all tests") - ds := CreateMySQLDS(t) - defer ds.Close() - setupPackSpecsTest(t, ds) host, err := ds.NewHost(context.Background(), &fleet.Host{ diff --git a/server/datastore/mysql/password_reset_test.go b/server/datastore/mysql/password_reset_test.go index 71c0c5ca62..ce9b2fcc40 100644 --- a/server/datastore/mysql/password_reset_test.go +++ b/server/datastore/mysql/password_reset_test.go @@ -9,11 +9,25 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPasswordResetRequests(t *testing.T) { - db := CreateMySQLDS(t) - defer db.Close() +func TestPasswordReset(t *testing.T) { + ds := CreateMySQLDS(t) - createTestUsers(t, db) + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Requests", testPasswordResetRequests}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testPasswordResetRequests(t *testing.T, ds *Datastore) { + createTestUsers(t, ds) now := time.Now().UTC() tomorrow := now.Add(time.Hour * 24) var passwordResetTests = []struct { @@ -30,7 +44,7 @@ func TestPasswordResetRequests(t *testing.T) { ExpiresAt: tt.expires, Token: tt.token, } - req, err := db.NewPasswordResetRequest(context.Background(), r) + req, err := ds.NewPasswordResetRequest(context.Background(), r) assert.Nil(t, err) assert.Equal(t, tt.userID, req.UserID) } diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index a5fb49b7eb..2ea27f71e0 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -12,10 +12,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewGlobalPolicy(t *testing.T) { +func TestPolicies(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"NewGlobalPolicy", testPoliciesNewGlobalPolicy}, + {"MembershipView", testPoliciesMembershipView}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testPoliciesNewGlobalPolicy(t *testing.T, ds *Datastore) { q, err := ds.NewQuery(context.Background(), &fleet.Query{ Name: "query1", Description: "query1 desc", @@ -58,10 +73,7 @@ func TestNewGlobalPolicy(t *testing.T) { require.NoError(t, ds.DeleteQuery(context.Background(), q.Name)) } -func TestPolicyMembershipView(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testPoliciesMembershipView(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), diff --git a/server/datastore/mysql/queries_test.go b/server/datastore/mysql/queries_test.go index c080289a81..5217522ff5 100644 --- a/server/datastore/mysql/queries_test.go +++ b/server/datastore/mysql/queries_test.go @@ -12,10 +12,31 @@ import ( "github.com/stretchr/testify/require" ) -func TestApplyQueries(t *testing.T) { +func TestQueries(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Apply", testQueriesApply}, + {"Delete", testQueriesDelete}, + {"GetByName", testQueriesGetByName}, + {"DeleteMany", testQueriesDeleteMany}, + {"Save", testQueriesSave}, + {"List", testQueriesList}, + {"LoadPacksForQueries", testQueriesLoadPacksForQueries}, + {"DuplicateNew", testQueriesDuplicateNew}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testQueriesApply(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) @@ -79,10 +100,7 @@ func TestApplyQueries(t *testing.T) { assert.Equal(t, &zwass.ID, queries[2].AuthorID) } -func TestDeleteQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesDelete(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) query := &fleet.Query{ @@ -103,10 +121,7 @@ func TestDeleteQuery(t *testing.T) { assert.NotNil(t, err) } -func TestGetQueryByName(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesGetByName(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) actual, err := ds.QueryByName(context.Background(), "q1") @@ -119,10 +134,7 @@ func TestGetQueryByName(t *testing.T) { assert.True(t, fleet.IsNotFound(err)) } -func TestDeleteQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesDeleteMany(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) q1 := test.NewQuery(t, ds, "q1", "select * from time", user.ID, true) @@ -157,13 +169,9 @@ func TestDeleteQueries(t *testing.T) { queries, err = ds.ListQueries(context.Background(), fleet.ListQueryOptions{}) require.Nil(t, err) assert.Len(t, queries, 0) - } -func TestSaveQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesSave(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) query := &fleet.Query{ @@ -190,10 +198,7 @@ func TestSaveQuery(t *testing.T) { assert.True(t, queryVerify.ObserverCanRun) } -func TestListQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesList(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) for i := 0; i < 10; i++ { @@ -221,10 +226,7 @@ func TestListQuery(t *testing.T) { assert.Equal(t, 10, len(results)) } -func TestLoadPacksForQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesLoadPacksForQueries(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "q1", Query: "select * from time"}, @@ -348,10 +350,7 @@ func TestLoadPacksForQueries(t *testing.T) { } } -func TestDuplicateNewQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testQueriesDuplicateNew(t *testing.T, ds *Datastore) { user := test.NewUser(t, ds, "Mike Arpaia", "mike@fleet.co", true) q1, err := ds.NewQuery(context.Background(), &fleet.Query{ Name: "foo", diff --git a/server/datastore/mysql/scheduled_queries_test.go b/server/datastore/mysql/scheduled_queries_test.go index 8345c08118..a6268c665b 100644 --- a/server/datastore/mysql/scheduled_queries_test.go +++ b/server/datastore/mysql/scheduled_queries_test.go @@ -12,10 +12,29 @@ import ( "github.com/stretchr/testify/require" ) -func TestListScheduledQueriesInPack(t *testing.T) { +func TestScheduledQueries(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"ListInPack", testScheduledQueriesListInPack}, + {"New", testScheduledQueriesNew}, + {"Get", testScheduledQueriesGet}, + {"Delete", testScheduledQueriesDelete}, + {"CascadingDelete", testScheduledQueriesCascadingDelete}, + {"CleanupOrphanStats", testScheduledQueriesCleanupOrphanStats}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testScheduledQueriesListInPack(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "foo", Description: "get the foos", Query: "select * from foo"}, @@ -81,10 +100,7 @@ func TestListScheduledQueriesInPack(t *testing.T) { require.Len(t, gotQueries, 3) } -func TestNewScheduledQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesNew(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -100,10 +116,7 @@ func TestNewScheduledQuery(t *testing.T) { assert.Equal(t, "select * from time;", query.Query) } -func TestScheduledQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesGet(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -127,10 +140,7 @@ func TestScheduledQuery(t *testing.T) { assert.False(t, *query.Denylist) } -func TestDeleteScheduledQuery(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesDelete(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") @@ -147,10 +157,7 @@ func TestDeleteScheduledQuery(t *testing.T) { require.NotNil(t, err) } -func TestCascadingDeletionOfQueries(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesCascadingDelete(t *testing.T, ds *Datastore) { zwass := test.NewUser(t, ds, "Zach", "zwass@fleet.co", true) queries := []*fleet.Query{ {Name: "foo", Description: "get the foos", Query: "select * from foo"}, @@ -199,10 +206,7 @@ func TestCascadingDeletionOfQueries(t *testing.T) { require.Len(t, gotQueries, 1) } -func TestCleanupOrphanScheduledQueryStats(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testScheduledQueriesCleanupOrphanStats(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, "Admin", "admin@fleet.co", true) q1 := test.NewQuery(t, ds, "foo", "select * from time;", u1.ID, true) p1 := test.NewPack(t, ds, "baz") diff --git a/server/datastore/mysql/sessions_test.go b/server/datastore/mysql/sessions_test.go index 14c225a8a8..446052e653 100644 --- a/server/datastore/mysql/sessions_test.go +++ b/server/datastore/mysql/sessions_test.go @@ -10,10 +10,24 @@ import ( "github.com/stretchr/testify/require" ) -func TestSessionGetters(t *testing.T) { +func TestSessions(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Getters", testSessionsGetters}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testSessionsGetters(t *testing.T, ds *Datastore) { user, err := ds.NewUser(context.Background(), &fleet.User{ Password: []byte("supersecret"), Email: "other@bobcom", diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 04952635bf..c8d754e526 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -332,11 +332,18 @@ func (d *Datastore) AllSoftwareWithoutCPEIterator(ctx context.Context) (fleet.So } func (d *Datastore) AddCPEForSoftware(ctx context.Context, software fleet.Software, cpe string) error { + _, err := addCPEForSoftwareDB(ctx, d.writer, software, cpe) + return err +} + +func addCPEForSoftwareDB(ctx context.Context, exec sqlx.ExecerContext, software fleet.Software, cpe string) (uint, error) { sql := `INSERT INTO software_cpe (software_id, cpe) VALUES (?, ?)` - if _, err := d.writer.ExecContext(ctx, sql, software.ID, cpe); err != nil { - return errors.Wrap(err, "insert software cpe") + res, err := exec.ExecContext(ctx, sql, software.ID, cpe) + if err != nil { + return 0, errors.Wrap(err, "insert software cpe") } - return nil + id, _ := res.LastInsertId() // cannot fail with the mysql driver + return uint(id), nil } func (d *Datastore) AllCPEs(ctx context.Context) ([]string, error) { diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 9e52730939..099bdeb5a1 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -11,14 +11,37 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/test" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestSaveHostSoftware(t *testing.T) { +func TestSoftware(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"SaveHost", testSoftwareSaveHost}, + {"CPE", testSoftwareCPE}, + {"InsertCVEs", testSoftwareInsertCVEs}, + {"HostDuplicates", testSoftwareHostDuplicates}, + {"LoadVulnerabilities", testSoftwareLoadVulnerabilities}, + {"AllCPEs", testSoftwareAllCPEs}, + {"NothingChanged", testSoftwareNothingChanged}, + {"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs}, + {"List", testSoftwareList}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testSoftwareSaveHost(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) @@ -103,10 +126,7 @@ func TestSaveHostSoftware(t *testing.T) { test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software) } -func TestSoftwareCPE(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareCPE(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft1 := fleet.HostSoftware{ @@ -176,10 +196,7 @@ func TestSoftwareCPE(t *testing.T) { require.NoError(t, iterator.Close()) } -func TestInsertCVEs(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareInsertCVEs(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -197,10 +214,7 @@ func TestInsertCVEs(t *testing.T) { require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) } -func TestHostSoftwareDuplicates(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) longName := strings.Repeat("a", 260) @@ -232,10 +246,7 @@ func TestHostSoftwareDuplicates(t *testing.T) { require.NoError(t, tx.Commit()) } -func TestLoadSoftwareVulnerabilities(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -270,10 +281,7 @@ func TestLoadSoftwareVulnerabilities(t *testing.T) { require.Len(t, host.Software[1].Vulnerabilities, 0) } -func TestAllCPEs(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareAllCPEs(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -296,7 +304,7 @@ func TestAllCPEs(t *testing.T) { assert.ElementsMatch(t, cpes, []string{"somecpe", "someothercpewithoutvulns"}) } -func TestNothingChanged(t *testing.T) { +func testSoftwareNothingChanged(t *testing.T, ds *Datastore) { assert.False(t, nothingChanged(nil, []fleet.Software{{}})) assert.True(t, nothingChanged(nil, nil)) assert.True(t, nothingChanged( @@ -313,10 +321,7 @@ func TestNothingChanged(t *testing.T) { )) } -func TestLoadSupportsTonsOfCVEs(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareLoadSupportsTonsOfCVEs(t *testing.T, ds *Datastore) { host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) soft := fleet.HostSoftware{ @@ -332,15 +337,26 @@ func TestLoadSupportsTonsOfCVEs(t *testing.T) { require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) sort.Slice(host.Software, func(i, j int) bool { return host.Software[i].Name < host.Software[j].Name }) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe")) + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns")) - for i := 0; i < 1000; i++ { - part1 := rand.Intn(1000) - part2 := rand.Intn(1000) - part3 := rand.Intn(1000) - cve := fmt.Sprintf("cve-%d-%d-%d", part1, part2, part3) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), cve, []string{"somecpe"})) - } + + require.NoError(t, ds.withTx(context.Background(), func(tx sqlx.ExtContext) error { + somecpeID, err := addCPEForSoftwareDB(context.Background(), tx, host.Software[0], "somecpe") + if err != nil { + return err + } + sql := `INSERT IGNORE INTO software_cve (cpe_id, cve) VALUES (?, ?)` + for i := 0; i < 1000; i++ { + part1 := rand.Intn(1000) + part2 := rand.Intn(1000) + part3 := rand.Intn(1000) + cve := fmt.Sprintf("cve-%d-%d-%d", part1, part2, part3) + if _, err := tx.ExecContext(context.Background(), sql, somecpeID, cve); err != nil { + return err + } + } + return nil + })) require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) @@ -363,10 +379,7 @@ func TestLoadSupportsTonsOfCVEs(t *testing.T) { } } -func TestListSoftware(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testSoftwareList(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) diff --git a/server/datastore/mysql/statistics_test.go b/server/datastore/mysql/statistics_test.go index 201175e7c2..420effa5b9 100644 --- a/server/datastore/mysql/statistics_test.go +++ b/server/datastore/mysql/statistics_test.go @@ -10,10 +10,24 @@ import ( "github.com/stretchr/testify/require" ) -func TestShouldSendStatistics(t *testing.T) { +func TestStatistics(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"ShouldSend", testStatisticsShouldSend}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testStatisticsShouldSend(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), diff --git a/server/datastore/mysql/targets_test.go b/server/datastore/mysql/targets_test.go index dabd73a76d..b3b8957e37 100644 --- a/server/datastore/mysql/targets_test.go +++ b/server/datastore/mysql/targets_test.go @@ -14,10 +14,27 @@ import ( "github.com/stretchr/testify/require" ) -func TestCountHostsInTargets(t *testing.T) { +func TestTargets(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"CountHosts", testTargetsCountHosts}, + {"HostStatus", testTargetsHostStatus}, + {"HostIDsInTargets", testTargetsHostIDsInTargets}, + {"HostIDsInTargetsTeam", testTargetsHostIDsInTargetsTeam}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testTargetsCountHosts(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -156,13 +173,9 @@ func TestCountHostsInTargets(t *testing.T) { assert.Equal(t, uint(0), metrics.OnlineHosts) assert.Equal(t, uint(5), metrics.OfflineHosts) assert.Equal(t, uint(1), metrics.MissingInActionHosts) - } -func TestHostStatus(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTargetsHostStatus(t *testing.T, ds *Datastore) { test.AddAllHostsLabel(t, ds) mockClock := clock.NewMockClock() @@ -222,10 +235,7 @@ func TestHostStatus(t *testing.T) { } } -func TestHostIDsInTargets(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTargetsHostIDsInTargets(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} @@ -302,10 +312,7 @@ func TestHostIDsInTargets(t *testing.T) { assert.Equal(t, []uint{1, 3, 4, 5, 6}, ids) } -func TestHostIDsInTargetsTeam(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) { user := &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)} filter := fleet.TeamFilter{User: user} diff --git a/server/datastore/mysql/teams_test.go b/server/datastore/mysql/teams_test.go index 21bb004915..744299f0b9 100644 --- a/server/datastore/mysql/teams_test.go +++ b/server/datastore/mysql/teams_test.go @@ -13,10 +13,28 @@ import ( "github.com/stretchr/testify/require" ) -func TestTeamGetSetDelete(t *testing.T) { +func TestTeams(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"GetSetDelete", testTeamsGetSetDelete}, + {"Users", testTeamsUsers}, + {"List", testTeamsList}, + {"Search", testTeamsSearch}, + {"EnrollSecrets", testTeamsEnrollSecrets}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testTeamsGetSetDelete(t *testing.T, ds *Datastore) { var createTests = []struct { name, description string }{ @@ -25,7 +43,7 @@ func TestTeamGetSetDelete(t *testing.T) { } for _, tt := range createTests { - t.Run("", func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { team, err := ds.NewTeam(context.Background(), &fleet.Team{ Name: tt.name, Description: tt.description, @@ -52,10 +70,7 @@ func TestTeamGetSetDelete(t *testing.T) { } } -func TestTeamUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsUsers(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) user1 := fleet.User{Name: users[0].Name, Email: users[0].Email, ID: users[0].ID} user2 := fleet.User{Name: users[1].Name, Email: users[1].Email, ID: users[1].ID} @@ -111,10 +126,7 @@ func TestTeamUsers(t *testing.T) { assert.ElementsMatch(t, team2Users, team2.Users) } -func TestTeamListTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsList(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) user1 := fleet.User{Name: users[0].Name, Email: users[0].Email, ID: users[0].ID, GlobalRole: ptr.String(fleet.RoleAdmin)} user2 := fleet.User{Name: users[1].Name, Email: users[1].Email, ID: users[1].ID, GlobalRole: ptr.String(fleet.RoleObserver)} @@ -168,10 +180,7 @@ func TestTeamListTeams(t *testing.T) { assert.Equal(t, 1, teams[1].UserCount) } -func TestTeamSearchTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsSearch(t *testing.T, ds *Datastore) { team1, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team1"}) require.NoError(t, err) team2, err := ds.NewTeam(context.Background(), &fleet.Team{Name: "team2"}) @@ -207,10 +216,7 @@ func TestTeamSearchTeams(t *testing.T) { assert.Len(t, teams, 0) } -func TestTeamEnrollSecrets(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testTeamsEnrollSecrets(t *testing.T, ds *Datastore) { secrets := []*fleet.EnrollSecret{{Secret: "secret1"}, {Secret: "secret2"}} team1, err := ds.NewTeam(context.Background(), &fleet.Team{ Name: "team1", diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index be4d04a80e..e90a9a3302 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -233,7 +233,9 @@ func createMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datasto strings.TrimPrefix(details.Name(), "github.com/fleetdm/fleet/v4/"), "/", "_", ) cleanName = strings.ReplaceAll(cleanName, ".", "_") - return initializeDatabase(t, cleanName, opts) + ds := initializeDatabase(t, cleanName, opts) + t.Cleanup(func() { ds.Close() }) + return ds } func CreateMySQLDSWithOptions(t *testing.T, opts *DatastoreTestOptions) *Datastore { @@ -250,5 +252,64 @@ func CreateNamedMySQLDS(t *testing.T, name string) *Datastore { } t.Parallel() - return initializeDatabase(t, name, new(DatastoreTestOptions)) + ds := initializeDatabase(t, name, new(DatastoreTestOptions)) + t.Cleanup(func() { ds.Close() }) + return ds +} + +// TruncateTables truncates the specified tables, in order, using ds.writer. +// Note that the order is typically not important because FK checks are +// disabled while truncating. If no table is provided, all tables (except +// those that are seeded by the SQL schema file) are truncated. +func TruncateTables(t *testing.T, ds *Datastore, tables ...string) { + // those tables are seeded with the schema.sql and as such must not + // be truncated - a more precise approach must be used for those, e.g. + // delete where id > max before test, or something like that. + nonEmptyTables := map[string]bool{ + "app_config_json": true, + "app_configs": true, + "migration_status_tables": true, + "osquery_options": true, + } + + ctx := context.Background() + + require.NoError(t, ds.withTx(ctx, func(tx sqlx.ExtContext) error { + var skipSeeded bool + + if len(tables) == 0 { + skipSeeded = true + sql := ` + SELECT + table_name + FROM + information_schema.tables + WHERE + table_schema = database() AND + table_type = 'BASE TABLE' + ` + if err := sqlx.SelectContext(ctx, tx, &tables, sql); err != nil { + return err + } + } + + if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=0`); err != nil { + return err + } + for _, tbl := range tables { + if nonEmptyTables[tbl] { + if skipSeeded { + continue + } + return fmt.Errorf("cannot truncate table %s, it contains seed data from schema.sql", tbl) + } + if _, err := tx.ExecContext(ctx, "TRUNCATE TABLE "+tbl); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=1`); err != nil { + return err + } + return nil + })) } diff --git a/server/datastore/mysql/users_test.go b/server/datastore/mysql/users_test.go index 86562c97f6..240e2a338e 100644 --- a/server/datastore/mysql/users_test.go +++ b/server/datastore/mysql/users_test.go @@ -15,10 +15,30 @@ import ( "github.com/stretchr/testify/require" ) -func TestCreateUser(t *testing.T) { +func TestUsers(t *testing.T) { ds := CreateMySQLDS(t) - defer ds.Close() + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Create", testUsersCreate}, + {"ByID", testUsersByID}, + {"Save", testUsersSave}, + {"List", testUsersList}, + {"Teams", testUsersTeams}, + {"CreateWithTeams", testUsersCreateWithTeams}, + {"SaveMany", testUsersSaveMany}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testUsersCreate(t *testing.T, ds *Datastore) { var createTests = []struct { password, email string isAdmin, passwordReset, sso bool @@ -48,10 +68,7 @@ func TestCreateUser(t *testing.T) { } } -func TestUserByID(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersByID(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) for _, tt := range users { returned, err := ds.UserByID(context.Background(), tt.ID) @@ -92,10 +109,7 @@ func createTestUsers(t *testing.T, ds fleet.Datastore) []*fleet.User { return users } -func TestSaveUser(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersSave(t *testing.T, ds *Datastore) { users := createTestUsers(t, ds) testUserGlobalRole(t, ds, users) testEmailAttribute(t, ds, users) @@ -150,10 +164,7 @@ func testUserGlobalRole(t *testing.T, ds fleet.Datastore, users []*fleet.User) { assert.Equal(t, "Cannot specify both Global Role and Team Roles", flErr.Message) } -func TestListUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersList(t *testing.T, ds *Datastore) { createTestUsers(t, ds) users, err := ds.ListUsers(context.Background(), fleet.UserListOptions{}) @@ -171,10 +182,7 @@ func TestListUsers(t *testing.T) { assert.Equal(t, "mike@fleet.co", users[0].Email) } -func TestUserTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersTeams(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -264,10 +272,7 @@ func TestUserTeams(t *testing.T) { assert.Len(t, users[1].Teams, 0) } -func TestUserCreateWithTeams(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersCreateWithTeams(t *testing.T, ds *Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewTeam(context.Background(), &fleet.Team{Name: fmt.Sprintf("%d", i)}) require.NoError(t, err) @@ -306,10 +311,7 @@ func TestUserCreateWithTeams(t *testing.T) { assert.Equal(t, "maintainer", user.Teams[2].Role) } -func TestSaveUsers(t *testing.T) { - ds := CreateMySQLDS(t) - defer ds.Close() - +func testUsersSaveMany(t *testing.T, ds *Datastore) { u1 := test.NewUser(t, ds, t.Name()+"Admin1", t.Name()+"admin1@fleet.co", true) u2 := test.NewUser(t, ds, t.Name()+"Admin2", t.Name()+"admin2@fleet.co", true) u3 := test.NewUser(t, ds, t.Name()+"Admin3", t.Name()+"admin3@fleet.co", true) diff --git a/server/vulnerabilities/cpe_test.go b/server/vulnerabilities/cpe_test.go index e0e35e9170..ed004902bf 100644 --- a/server/vulnerabilities/cpe_test.go +++ b/server/vulnerabilities/cpe_test.go @@ -53,6 +53,10 @@ func TestCpeFromSoftware(t *testing.T) { } func TestSyncCPEDatabase(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + // Disabling vcr because the resulting file exceeds the 100mb limit for github r, err := recorder.NewAsMode("fixtures/nvd-cpe-release", recorder.ModeDisabled, http.DefaultTransport) require.NoError(t, err) @@ -144,6 +148,10 @@ func (f *fakeSoftwareIterator) Err() error { return nil } func (f *fakeSoftwareIterator) Close() error { f.closed = true; return nil } func TestTranslateSoftwareToCPE(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + tempDir, err := os.MkdirTemp(os.TempDir(), "TestTranslateSoftwareToCPE-*") require.NoError(t, err) defer os.RemoveAll(tempDir) diff --git a/server/vulnerabilities/cve_test.go b/server/vulnerabilities/cve_test.go index adc83e8082..cbf984b600 100644 --- a/server/vulnerabilities/cve_test.go +++ b/server/vulnerabilities/cve_test.go @@ -26,6 +26,10 @@ var cvetests = []struct { } func TestTranslateCPEToCVE(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + tempDir := t.TempDir() ds := new(mock.Store) From dd31779aacda1883521b1dd752f84bce4381c0b6 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Mon, 20 Sep 2021 11:13:55 -0700 Subject: [PATCH 28/82] Increase timeout for golangci-lint (#2143) --- .github/workflows/golangci-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d1072848fc..49be7f82d4 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -17,4 +17,4 @@ jobs: # specified without patch version: we always use the latest patch # version. version: v1.42 - args: --timeout 3m + args: --timeout 10m From 5eb14e290f01ced6df7dcf07511d41b3efa14dad Mon Sep 17 00:00:00 2001 From: eashaw Date: Mon, 20 Sep 2021 13:29:59 -0500 Subject: [PATCH 29/82] Fix 500 error being thrown on fleetdm.com (#2144) * update notFound conditional * Update view-basic-documentation.js --- website/api/controllers/docs/view-basic-documentation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/api/controllers/docs/view-basic-documentation.js b/website/api/controllers/docs/view-basic-documentation.js index 525bee33c2..463d134991 100644 --- a/website/api/controllers/docs/view-basic-documentation.js +++ b/website/api/controllers/docs/view-basic-documentation.js @@ -57,7 +57,7 @@ module.exports = { let revisedPage = _.find(sails.config.builtStaticContent.markdownPages, { url: _.trimRight(SECTION_URL_PREFIX + '/' + _.trim(modifiedPageUrlSuffix, '/'), '/') }); - if(revisedPage.url) { + if(revisedPage) { // If we matched a page with the modified pageUrlSuffix, then redirect to that. throw {redirect: revisedPage.url}; } else { From 4f3f6187d6a3c2c2b05aa53b6c0b29308770de19 Mon Sep 17 00:00:00 2001 From: Martavis Parker <47053705+martavis@users.noreply.github.com> Date: Mon, 20 Sep 2021 11:48:24 -0700 Subject: [PATCH 30/82] Top-level seed data doc and re-numbering (#2109) * created separate doc for seeding data * re-numbered doc names --- .../00-Learn-how-to-use-Fleet.md} | 0 .../01-Fleet-UI.md} | 0 .../02-fleetctl-CLI.md} | 0 .../03-REST-API.md} | 0 .../04-Adding-hosts.md} | 0 .../05-Osquery-logs.md} | 0 .../06-Monitoring-Fleet.md} | 0 .../07-Security-best-practices.md} | 0 .../08-Updating-Fleet.md} | 0 .../09-Permissions.md} | 0 .../10-Teams.md | 0 .../11-Usage-statistics.md | 0 .../12-Supported-browsers.md | 0 .../13-Vulnerability-Processing.md | 0 docs/{1-Using-Fleet => 01-Using-Fleet}/FAQ.md | 0 .../README.md | 0 .../configuration-files/README.md | 0 .../kubernetes/fleet-deployment.yml | 0 .../kubernetes/fleet-migrations.yml | 0 .../kubernetes/fleet-service.yml | 38 +++++++-------- .../enroll-secrets.yml | 0 .../multi-file-configuration/labels.yml | 0 .../organization-settings.yml | 0 .../multi-file-configuration/pack.yml | 0 .../multi-file-configuration/queries.yml | 0 .../single-file-configuration.yml | 0 .../standard-query-library/README.md | 0 .../standard-query-library.yml | 0 .../01-Installation.md} | 0 .../02-Configuration.md} | 0 .../03-Example-deployment-scenarios.md} | 0 .../04-fleetctl-agent-updates.md} | 0 docs/{2-Deploying => 02-Deploying}/FAQ.md | 0 docs/{2-Deploying => 02-Deploying}/README.md | 0 .../01-Building-Fleet.md} | 0 .../02-Testing.md} | 47 +------------------ .../03-Migrations.md} | 0 .../04-Committing-Changes.md} | 0 .../05-Releasing-Fleet.md} | 0 docs/03-Contributing/06-Seeding-Data.md | 44 +++++++++++++++++ .../FAQ.md | 0 .../README.md | 0 42 files changed, 64 insertions(+), 65 deletions(-) rename docs/{1-Using-Fleet/0-Learn-how-to-use-Fleet.md => 01-Using-Fleet/00-Learn-how-to-use-Fleet.md} (100%) rename docs/{1-Using-Fleet/1-Fleet-UI.md => 01-Using-Fleet/01-Fleet-UI.md} (100%) rename docs/{1-Using-Fleet/2-fleetctl-CLI.md => 01-Using-Fleet/02-fleetctl-CLI.md} (100%) rename docs/{1-Using-Fleet/3-REST-API.md => 01-Using-Fleet/03-REST-API.md} (100%) rename docs/{1-Using-Fleet/4-Adding-hosts.md => 01-Using-Fleet/04-Adding-hosts.md} (100%) rename docs/{1-Using-Fleet/5-Osquery-logs.md => 01-Using-Fleet/05-Osquery-logs.md} (100%) rename docs/{1-Using-Fleet/6-Monitoring-Fleet.md => 01-Using-Fleet/06-Monitoring-Fleet.md} (100%) rename docs/{1-Using-Fleet/7-Security-best-practices.md => 01-Using-Fleet/07-Security-best-practices.md} (100%) rename docs/{1-Using-Fleet/8-Updating-Fleet.md => 01-Using-Fleet/08-Updating-Fleet.md} (100%) rename docs/{1-Using-Fleet/9-Permissions.md => 01-Using-Fleet/09-Permissions.md} (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/10-Teams.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/11-Usage-statistics.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/12-Supported-browsers.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/13-Vulnerability-Processing.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/FAQ.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/README.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/README.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/kubernetes/fleet-deployment.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/kubernetes/fleet-migrations.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/kubernetes/fleet-service.yml (94%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/multi-file-configuration/enroll-secrets.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/multi-file-configuration/labels.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/multi-file-configuration/organization-settings.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/multi-file-configuration/pack.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/multi-file-configuration/queries.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/configuration-files/single-file-configuration.yml (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/standard-query-library/README.md (100%) rename docs/{1-Using-Fleet => 01-Using-Fleet}/standard-query-library/standard-query-library.yml (100%) rename docs/{2-Deploying/1-Installation.md => 02-Deploying/01-Installation.md} (100%) rename docs/{2-Deploying/2-Configuration.md => 02-Deploying/02-Configuration.md} (100%) rename docs/{2-Deploying/3-Example-deployment-scenarios.md => 02-Deploying/03-Example-deployment-scenarios.md} (100%) rename docs/{2-Deploying/4-fleetctl-agent-updates.md => 02-Deploying/04-fleetctl-agent-updates.md} (100%) rename docs/{2-Deploying => 02-Deploying}/FAQ.md (100%) rename docs/{2-Deploying => 02-Deploying}/README.md (100%) rename docs/{3-Contributing/1-Building-Fleet.md => 03-Contributing/01-Building-Fleet.md} (100%) rename docs/{3-Contributing/2-Testing.md => 03-Contributing/02-Testing.md} (80%) rename docs/{3-Contributing/3-Migrations.md => 03-Contributing/03-Migrations.md} (100%) rename docs/{3-Contributing/4-Committing-Changes.md => 03-Contributing/04-Committing-Changes.md} (100%) rename docs/{3-Contributing/5-Releasing-Fleet.md => 03-Contributing/05-Releasing-Fleet.md} (100%) create mode 100644 docs/03-Contributing/06-Seeding-Data.md rename docs/{3-Contributing => 03-Contributing}/FAQ.md (100%) rename docs/{3-Contributing => 03-Contributing}/README.md (100%) diff --git a/docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md b/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md similarity index 100% rename from docs/1-Using-Fleet/0-Learn-how-to-use-Fleet.md rename to docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md diff --git a/docs/1-Using-Fleet/1-Fleet-UI.md b/docs/01-Using-Fleet/01-Fleet-UI.md similarity index 100% rename from docs/1-Using-Fleet/1-Fleet-UI.md rename to docs/01-Using-Fleet/01-Fleet-UI.md diff --git a/docs/1-Using-Fleet/2-fleetctl-CLI.md b/docs/01-Using-Fleet/02-fleetctl-CLI.md similarity index 100% rename from docs/1-Using-Fleet/2-fleetctl-CLI.md rename to docs/01-Using-Fleet/02-fleetctl-CLI.md diff --git a/docs/1-Using-Fleet/3-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md similarity index 100% rename from docs/1-Using-Fleet/3-REST-API.md rename to docs/01-Using-Fleet/03-REST-API.md diff --git a/docs/1-Using-Fleet/4-Adding-hosts.md b/docs/01-Using-Fleet/04-Adding-hosts.md similarity index 100% rename from docs/1-Using-Fleet/4-Adding-hosts.md rename to docs/01-Using-Fleet/04-Adding-hosts.md diff --git a/docs/1-Using-Fleet/5-Osquery-logs.md b/docs/01-Using-Fleet/05-Osquery-logs.md similarity index 100% rename from docs/1-Using-Fleet/5-Osquery-logs.md rename to docs/01-Using-Fleet/05-Osquery-logs.md diff --git a/docs/1-Using-Fleet/6-Monitoring-Fleet.md b/docs/01-Using-Fleet/06-Monitoring-Fleet.md similarity index 100% rename from docs/1-Using-Fleet/6-Monitoring-Fleet.md rename to docs/01-Using-Fleet/06-Monitoring-Fleet.md diff --git a/docs/1-Using-Fleet/7-Security-best-practices.md b/docs/01-Using-Fleet/07-Security-best-practices.md similarity index 100% rename from docs/1-Using-Fleet/7-Security-best-practices.md rename to docs/01-Using-Fleet/07-Security-best-practices.md diff --git a/docs/1-Using-Fleet/8-Updating-Fleet.md b/docs/01-Using-Fleet/08-Updating-Fleet.md similarity index 100% rename from docs/1-Using-Fleet/8-Updating-Fleet.md rename to docs/01-Using-Fleet/08-Updating-Fleet.md diff --git a/docs/1-Using-Fleet/9-Permissions.md b/docs/01-Using-Fleet/09-Permissions.md similarity index 100% rename from docs/1-Using-Fleet/9-Permissions.md rename to docs/01-Using-Fleet/09-Permissions.md diff --git a/docs/1-Using-Fleet/10-Teams.md b/docs/01-Using-Fleet/10-Teams.md similarity index 100% rename from docs/1-Using-Fleet/10-Teams.md rename to docs/01-Using-Fleet/10-Teams.md diff --git a/docs/1-Using-Fleet/11-Usage-statistics.md b/docs/01-Using-Fleet/11-Usage-statistics.md similarity index 100% rename from docs/1-Using-Fleet/11-Usage-statistics.md rename to docs/01-Using-Fleet/11-Usage-statistics.md diff --git a/docs/1-Using-Fleet/12-Supported-browsers.md b/docs/01-Using-Fleet/12-Supported-browsers.md similarity index 100% rename from docs/1-Using-Fleet/12-Supported-browsers.md rename to docs/01-Using-Fleet/12-Supported-browsers.md diff --git a/docs/1-Using-Fleet/13-Vulnerability-Processing.md b/docs/01-Using-Fleet/13-Vulnerability-Processing.md similarity index 100% rename from docs/1-Using-Fleet/13-Vulnerability-Processing.md rename to docs/01-Using-Fleet/13-Vulnerability-Processing.md diff --git a/docs/1-Using-Fleet/FAQ.md b/docs/01-Using-Fleet/FAQ.md similarity index 100% rename from docs/1-Using-Fleet/FAQ.md rename to docs/01-Using-Fleet/FAQ.md diff --git a/docs/1-Using-Fleet/README.md b/docs/01-Using-Fleet/README.md similarity index 100% rename from docs/1-Using-Fleet/README.md rename to docs/01-Using-Fleet/README.md diff --git a/docs/1-Using-Fleet/configuration-files/README.md b/docs/01-Using-Fleet/configuration-files/README.md similarity index 100% rename from docs/1-Using-Fleet/configuration-files/README.md rename to docs/01-Using-Fleet/configuration-files/README.md diff --git a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml rename to docs/01-Using-Fleet/configuration-files/kubernetes/fleet-deployment.yml diff --git a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml rename to docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml diff --git a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml similarity index 94% rename from docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml rename to docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml index a26f40a92c..098270f021 100644 --- a/docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml +++ b/docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml @@ -1,19 +1,19 @@ -apiVersion: v1 -kind: Service -metadata: - name: fleet-loadbalancer - labels: - app: fleet-loadbalancer -spec: - type: LoadBalancer - ports: - - name: proxy-tls - port: 443 - targetPort: 443 - protocol: TCP - - name: proxy-http - port: 80 - targetPort: 80 - protocol: TCP - selector: - app: fleet-webserver +apiVersion: v1 +kind: Service +metadata: + name: fleet-loadbalancer + labels: + app: fleet-loadbalancer +spec: + type: LoadBalancer + ports: + - name: proxy-tls + port: 443 + targetPort: 443 + protocol: TCP + - name: proxy-http + port: 80 + targetPort: 80 + protocol: TCP + selector: + app: fleet-webserver diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/enroll-secrets.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/labels.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/labels.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/labels.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/labels.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/pack.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/pack.yml diff --git a/docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/queries.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml rename to docs/01-Using-Fleet/configuration-files/multi-file-configuration/queries.yml diff --git a/docs/1-Using-Fleet/configuration-files/single-file-configuration.yml b/docs/01-Using-Fleet/configuration-files/single-file-configuration.yml similarity index 100% rename from docs/1-Using-Fleet/configuration-files/single-file-configuration.yml rename to docs/01-Using-Fleet/configuration-files/single-file-configuration.yml diff --git a/docs/1-Using-Fleet/standard-query-library/README.md b/docs/01-Using-Fleet/standard-query-library/README.md similarity index 100% rename from docs/1-Using-Fleet/standard-query-library/README.md rename to docs/01-Using-Fleet/standard-query-library/README.md diff --git a/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml b/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml similarity index 100% rename from docs/1-Using-Fleet/standard-query-library/standard-query-library.yml rename to docs/01-Using-Fleet/standard-query-library/standard-query-library.yml diff --git a/docs/2-Deploying/1-Installation.md b/docs/02-Deploying/01-Installation.md similarity index 100% rename from docs/2-Deploying/1-Installation.md rename to docs/02-Deploying/01-Installation.md diff --git a/docs/2-Deploying/2-Configuration.md b/docs/02-Deploying/02-Configuration.md similarity index 100% rename from docs/2-Deploying/2-Configuration.md rename to docs/02-Deploying/02-Configuration.md diff --git a/docs/2-Deploying/3-Example-deployment-scenarios.md b/docs/02-Deploying/03-Example-deployment-scenarios.md similarity index 100% rename from docs/2-Deploying/3-Example-deployment-scenarios.md rename to docs/02-Deploying/03-Example-deployment-scenarios.md diff --git a/docs/2-Deploying/4-fleetctl-agent-updates.md b/docs/02-Deploying/04-fleetctl-agent-updates.md similarity index 100% rename from docs/2-Deploying/4-fleetctl-agent-updates.md rename to docs/02-Deploying/04-fleetctl-agent-updates.md diff --git a/docs/2-Deploying/FAQ.md b/docs/02-Deploying/FAQ.md similarity index 100% rename from docs/2-Deploying/FAQ.md rename to docs/02-Deploying/FAQ.md diff --git a/docs/2-Deploying/README.md b/docs/02-Deploying/README.md similarity index 100% rename from docs/2-Deploying/README.md rename to docs/02-Deploying/README.md diff --git a/docs/3-Contributing/1-Building-Fleet.md b/docs/03-Contributing/01-Building-Fleet.md similarity index 100% rename from docs/3-Contributing/1-Building-Fleet.md rename to docs/03-Contributing/01-Building-Fleet.md diff --git a/docs/3-Contributing/2-Testing.md b/docs/03-Contributing/02-Testing.md similarity index 80% rename from docs/3-Contributing/2-Testing.md rename to docs/03-Contributing/02-Testing.md index 6fc7e5f756..21e880f35d 100644 --- a/docs/3-Contributing/2-Testing.md +++ b/docs/03-Contributing/02-Testing.md @@ -7,7 +7,7 @@ - [Test hosts](#test-hosts) - [Email](#email) - [Database backup/restore](#database-backuprestore) -- [Teams seed data](#teams-seed-data) +- [Seeding Data](./6-Seeding-Data.md) - [MySQL shell](#mysql-shell) - [Testing SSO](#testing-sso) @@ -231,51 +231,6 @@ Restore: Note that a "restore" will replace the state of the development database with the state from the backup. -## Teams seed data - -When developing Fleet, it may be useful to create seed data that includes users and teams. - -Check out this Loom demo video that walks through creating teams seed data: -https://www.loom.com/share/1c41a1540e8f41328a7a6cfc56ad0a01 - -For a text-based walkthrough, check out the following steps: - -First, create a `env` file with the following contents: - -``` -export SERVER_URL=https://localhost:8080 # your fleet server url and port -export CURL_FLAGS='-k -s' # set insecure flag -export TOKEN=eyJhbGciOi... # your login token -``` - -Next, set the `FLEET_ENV_PATH` to point to the `env` file. This will let the scripts in the `fleet/` folder source the env file. - -``` -export FLEET_ENV_PATH=/Users/victor/fleet_env -``` - -Finally run one of the bash scripts located in the [/tools/api](../../tools/api/README.md) directory. - -The `fleet/create_free` script will generate an environment to roughly reflect an installation of Fleet Free. The script creates 3 users with different roles. - -``` -./tools/api/fleet/teams/create_free -``` - -The `fleet/create_premium` script will generate an environment to roughly reflect an installation of Fleet Premium. The script will create 2 teams 4 users with different roles. - -``` -./tools/api/fleet/teams/create_premium -``` - -The `fleet/create_figma` script will generate an environment to reflect the mockups in the Fleet EE (current) Figma file. The script creates 3 teams and 12 users with different roles. - -``` -./tools/api/fleet/teams/create_figma -``` - -Each user generated by the script has their password set to `user123#`. - ## MySQL shell Connect to the MySQL shell to view and interact directly with the contents of the development database. diff --git a/docs/3-Contributing/3-Migrations.md b/docs/03-Contributing/03-Migrations.md similarity index 100% rename from docs/3-Contributing/3-Migrations.md rename to docs/03-Contributing/03-Migrations.md diff --git a/docs/3-Contributing/4-Committing-Changes.md b/docs/03-Contributing/04-Committing-Changes.md similarity index 100% rename from docs/3-Contributing/4-Committing-Changes.md rename to docs/03-Contributing/04-Committing-Changes.md diff --git a/docs/3-Contributing/5-Releasing-Fleet.md b/docs/03-Contributing/05-Releasing-Fleet.md similarity index 100% rename from docs/3-Contributing/5-Releasing-Fleet.md rename to docs/03-Contributing/05-Releasing-Fleet.md diff --git a/docs/03-Contributing/06-Seeding-Data.md b/docs/03-Contributing/06-Seeding-Data.md new file mode 100644 index 0000000000..4c18effe4a --- /dev/null +++ b/docs/03-Contributing/06-Seeding-Data.md @@ -0,0 +1,44 @@ +# Seeding Data + +When developing Fleet, it may be useful to create seed data that includes users and teams. + +Check out this Loom demo video that walks through creating teams seed data: +https://www.loom.com/share/1c41a1540e8f41328a7a6cfc56ad0a01 + +For a text-based walkthrough, check out the following steps: + +First, create a `env` file with the following contents: + +``` +export SERVER_URL=https://localhost:8080 # your fleet server url and port +export CURL_FLAGS='-k -s' # set insecure flag +export TOKEN=eyJhbGciOi... # your login token +``` + +Next, set the `FLEET_ENV_PATH` to point to the `env` file. This will let the scripts in the `fleet/` folder source the env file. + +``` +export FLEET_ENV_PATH=/Users/victor/fleet_env +``` + +Finally run one of the bash scripts located in the [/tools/api](../../tools/api/README.md) directory. + +The `fleet/create_free` script will generate an environment to roughly reflect an installation of Fleet Free. The script creates 3 users with different roles. + +``` +./tools/api/fleet/teams/create_free +``` + +The `fleet/create_premium` script will generate an environment to roughly reflect an installation of Fleet Premium. The script will create 2 teams 4 users with different roles. + +``` +./tools/api/fleet/teams/create_premium +``` + +The `fleet/create_figma` script will generate an environment to reflect the mockups in the Fleet EE (current) Figma file. The script creates 3 teams and 12 users with different roles. + +``` +./tools/api/fleet/teams/create_figma +``` + +Each user generated by the script has their password set to `user123#`. \ No newline at end of file diff --git a/docs/3-Contributing/FAQ.md b/docs/03-Contributing/FAQ.md similarity index 100% rename from docs/3-Contributing/FAQ.md rename to docs/03-Contributing/FAQ.md diff --git a/docs/3-Contributing/README.md b/docs/03-Contributing/README.md similarity index 100% rename from docs/3-Contributing/README.md rename to docs/03-Contributing/README.md From e115358a51174b3a2b9f1b82866c146d152632e8 Mon Sep 17 00:00:00 2001 From: eashaw Date: Mon, 20 Sep 2021 15:42:54 -0500 Subject: [PATCH 31/82] Fix failing build script (#2147) * update path to query library * throw error if there is no YAML file --- website/scripts/build-static-content.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/scripts/build-static-content.js b/website/scripts/build-static-content.js index fa638ff4e5..93e55dd189 100644 --- a/website/scripts/build-static-content.js +++ b/website/scripts/build-static-content.js @@ -25,8 +25,8 @@ module.exports = { await sails.helpers.flow.simultaneously([ async()=>{// Parse query library from YAML and prepare to bake them into the Sails app's configuration. - let RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO = 'docs/1-Using-Fleet/standard-query-library/standard-query-library.yml'; - let yaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO)); + let RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO = 'docs/01-Using-Fleet/standard-query-library/standard-query-library.yml'; + let yaml = await sails.helpers.fs.read(path.join(topLvlRepoPath, RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO)).intercept('doesNotExist', (err)=>new Error(`Could not find standard query library YAML file at "${RELATIVE_PATH_TO_QUERY_LIBRARY_YML_IN_FLEET_REPO}". Was it accidentally moved? Raw error: `+err.message)); let queriesWithProblematicRemediations = []; let queriesWithProblematicContributors = []; From 008a093130815839e6fa4d9dbbd0e43a406f3343 Mon Sep 17 00:00:00 2001 From: eashaw Date: Mon, 20 Sep 2021 15:52:37 -0500 Subject: [PATCH 32/82] update link to use correct variable (#2149) --- website/views/pages/docs/basic-documentation.ejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/views/pages/docs/basic-documentation.ejs b/website/views/pages/docs/basic-documentation.ejs index 6db93b1b0c..56c97246b3 100644 --- a/website/views/pages/docs/basic-documentation.ejs +++ b/website/views/pages/docs/basic-documentation.ejs @@ -205,7 +205,7 @@

    Is there something missing?

    - If you notice something we’ve missed, or that could be improved, please click here to edit this page. + If you notice something we’ve missed, or that could be improved, please click here to edit this page.

    From bcb5288f7177795353f034554a13d1bb31b12918 Mon Sep 17 00:00:00 2001 From: noahtalerman <47070608+noahtalerman@users.noreply.github.com> Date: Mon, 20 Sep 2021 15:40:11 -0700 Subject: [PATCH 33/82] Add Policies feature to permissions documentation (#2153) --- docs/01-Using-Fleet/09-Permissions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/01-Using-Fleet/09-Permissions.md b/docs/01-Using-Fleet/09-Permissions.md index af3bc7c678..0795a57348 100644 --- a/docs/01-Using-Fleet/09-Permissions.md +++ b/docs/01-Using-Fleet/09-Permissions.md @@ -16,6 +16,8 @@ The following table depicts various permissions levels for each role. | ---------------------------------------------------- | -------- | ---------- | ----- | | Browse all hosts | ✅ | ✅ | ✅ | | Filter hosts using labels | ✅ | ✅ | ✅ | +| Browse all policies | ✅ | ✅ | ✅ | +| Filter hosts using policies | ✅ | ✅ | ✅ | | Target hosts using labels | ✅ | ✅ | ✅ | | Run saved queries as live queries against all hosts | ✅ | ✅ | ✅ | | Run custom queries as live queries against all hosts | | ✅ | ✅ | @@ -66,8 +68,11 @@ The following table depicts various permissions levels in a team. | Action | Observer | Maintainer | | ------------------------------------------------------------ | -------- | ---------- | | Browse hosts assigned to team | ✅ | ✅ | +| Browse policies for hosts assigned to team | ✅ | ✅ | +| Filter hosts assigned to team using policies | ✅ | ✅ | | Filter hosts assigned to team using labels | ✅ | ✅ | | Target hosts assigned to team using labels | ✅ | ✅ | +| Browse policies for hosts assigned to team | ✅ | ✅ | | Run saved queries as live queries on hosts assigned to team | ✅ | ✅ | | Run custom queries as live queries on hosts assigned to team | | ✅ | | Enroll hosts to member team | | ✅ | From 1d1e0b5f5f2c4d0dec64f59cd931f845e4951ede Mon Sep 17 00:00:00 2001 From: Martavis Parker <47053705+martavis@users.noreply.github.com> Date: Mon, 20 Sep 2021 17:20:46 -0700 Subject: [PATCH 34/82] Fixed style for sidebar (#2154) * fixed style for sidebar * changes file * Reduce minimum width of body-wrap to prevent right sidebar from overflowing min app width of 1200px Co-authored-by: Noah Talerman --- changes/2107-sidebar-style | 1 + .../side_panels/SecondarySidePanelContainer/_styles.scss | 2 +- frontend/styles/global/_global.scss | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/2107-sidebar-style diff --git a/changes/2107-sidebar-style b/changes/2107-sidebar-style new file mode 100644 index 0000000000..0a50d56a17 --- /dev/null +++ b/changes/2107-sidebar-style @@ -0,0 +1 @@ +- Fixed sidebar style \ No newline at end of file diff --git a/frontend/components/side_panels/SecondarySidePanelContainer/_styles.scss b/frontend/components/side_panels/SecondarySidePanelContainer/_styles.scss index 430d613e77..31d90c0979 100644 --- a/frontend/components/side_panels/SecondarySidePanelContainer/_styles.scss +++ b/frontend/components/side_panels/SecondarySidePanelContainer/_styles.scss @@ -1,9 +1,9 @@ .secondary-side-panel-container { - align-self: stretch; background-color: $core-white; box-sizing: border-box; border-left: 1px solid $ui-gray; overflow: auto; + min-width: $sidepanel-width; width: $sidepanel-width; padding: $pad-xxlarge; } diff --git a/frontend/styles/global/_global.scss b/frontend/styles/global/_global.scss index 586d4bb247..1e335648f7 100644 --- a/frontend/styles/global/_global.scss +++ b/frontend/styles/global/_global.scss @@ -55,11 +55,12 @@ a { border-radius: 3px; background-color: $core-white; border: solid 1px $core-white; - min-width: 910px; + min-width: 798px; } .has-sidebar { display: flex; + flex-grow: 1; & > *:first-child { flex-grow: 1; @@ -121,4 +122,4 @@ hr { margin-bottom: $pad-xlarge; border: none; border-bottom: 1px solid $ui-fleet-blue-15; -} \ No newline at end of file +} From 470889ba3af53279895a5bb7b2fc116624dc0f70 Mon Sep 17 00:00:00 2001 From: eashaw Date: Mon, 20 Sep 2021 20:59:45 -0500 Subject: [PATCH 35/82] Update code blocks in documentation (#2151) * updated css to be compatible with Chrome 87 and earlier * fixed JSON syntax code blocks, remove empty response data * Update code-blocks.less * fix broken links --- docs/01-Using-Fleet/03-REST-API.md | 134 +---- website/.sailsrc | 499 +++++++++++------- .../assets/styles/pages/docs/code-blocks.less | 4 +- 3 files changed, 342 insertions(+), 295 deletions(-) diff --git a/docs/01-Using-Fleet/03-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md index f08fa28b9d..dfe6d99ab7 100644 --- a/docs/01-Using-Fleet/03-REST-API.md +++ b/docs/01-Using-Fleet/03-REST-API.md @@ -31,7 +31,7 @@ Fleet is powered by a Go API server which serves three types of endpoints: ### fleetctl -Many of the operations that a user may wish to perform with an API are currently best performed via the [fleetctl](./2-fleetctl-CLI.md) tooling. These CLI tools allow updating of the osquery configuration entities, as well as performing live queries. +Many of the operations that a user may wish to perform with an API are currently best performed via the [fleetctl](./02-fleetctl-CLI.md) tooling. These CLI tools allow updating of the osquery configuration entities, as well as performing live queries. ### Current API @@ -263,8 +263,8 @@ Resets a user's password. Which user is determined by the password reset token u ```json { - "new_password": "abc123" - "new_password_confirmation": "abc123" + "new_password": "abc123", + "new_password_confirmation": "abc123", "password_reset_token": "UU5EK0JhcVpsRkY3NTdsaVliMEZDbHJ6TWdhK3oxQ1Q=" } ``` @@ -273,9 +273,6 @@ Resets a user's password. Which user is determined by the password reset token u `Status: 200` -```json -{} -``` --- @@ -451,9 +448,6 @@ This is the callback endpoint that the identity provider will use to send securi `Status: 200` -```json -{} -``` --- @@ -482,7 +476,7 @@ This is the callback endpoint that the identity provider will use to send securi | order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`. | | status | string | query | Indicates the status of the hosts to return. Can either be `new`, `online`, `offline`, or `mia`. | | query | string | query | Search query keywords. Searchable fields include `hostname`, `machine_serial`, `uuid`, and `ipv4`. | -| additional_info_filters | string | query | A comma-delimited list of fields to include in each host's additional information object. See [Fleet Configuration Options](../1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options) for an example configuration with hosts' additional information. Use `*` to get all stored fields. | +| additional_info_filters | string | query | A comma-delimited list of fields to include in each host's additional information object. See [Fleet Configuration Options](../01-Using-Fleet/02-fleetctl-CLI.md#fleet-configuration-options) for an example configuration with hosts' additional information. Use `*` to get all stored fields. | | team_id | integer | query | _Available in Fleet Premium_ Filters the users to only include users in the specified team. | | policy_id | integer | query | The ID of the policy to filter hosts by. `policy_response` must also be specified with `policy_id`. | | policy_response | string | query | Valid options are `passing` or `failing`. `policy_id` must also be specified with `policy_response`. | @@ -584,7 +578,7 @@ None. Returns the information of the specified host. -The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../2-Deploying/2-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. +The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../02-Deploying/02-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature. `GET /api/v1/fleet/hosts/{id}` @@ -805,9 +799,6 @@ Deletes the specified host from Fleet. Note that a deleted host will fail authen `Status: 200` -```json -{} -``` ### Refetch host @@ -829,9 +820,6 @@ Flags the host details to be refetched the next time the host checks in for live `Status: 200` -```json -{} -``` ### Transfer hosts to a team @@ -863,9 +851,6 @@ _Available in Fleet Premium_ `Status: 200` -```json -{} -``` ### Transfer hosts to a team by filter @@ -899,9 +884,6 @@ _Available in Fleet Premium_ `Status: 200` -```json -{} -``` --- @@ -1252,9 +1234,6 @@ Deletes the label specified by name. `Status: 200` -```json -{} -``` ### Delete label by ID @@ -1276,9 +1255,6 @@ Deletes the label specified by ID. `Status: 200` -```json -{} -``` ### Apply labels specs @@ -1327,9 +1303,6 @@ If the `label_membership_type` is set to `manual`, the `hosts` property must als `Status: 200` -```json -{} -``` ### Get labels specs @@ -1704,11 +1677,11 @@ Creates a user account without requiring an invitation, the user is enabled imme "teams": [ { "id": 2, - "role: "observer" + "role": "observer" }, { "id": 3, - "role: "maintainer" + "role": "maintainer" }, ] } @@ -1856,10 +1829,10 @@ Returns all information about a specific user. "teams": [ { "id": 1, - "role: "observer" + "role": "observer" }, { - "id": 2 + "id": 2, "role": "maintainer" } ] @@ -1882,15 +1855,15 @@ Returns all information about a specific user. "force_password_reset": false, "gravatar_url": "", "sso_enabled": false, - "global_role": "admin" + "global_role": "admin", "teams": [ { "id": 2, - "role: "observer" + "role": "observer" }, { "id": 3, - "role: "maintainer" + "role": "maintainer" }, ] } @@ -1917,9 +1890,6 @@ Delete the specified user from Fleet. `Status: 200` -```json -{} -``` ### Require password reset @@ -2027,9 +1997,6 @@ Deletes the selected user's sessions in Fleet. Also deletes the user's API token `Status: 200` -```json -{} -``` --- @@ -2086,9 +2053,6 @@ Deletes the session specified by ID. When the user associated with the session n `Status: 200` -```json -{} -``` --- @@ -2351,9 +2315,6 @@ Deletes the query specified by name. `Status: 200` -```json -{} -``` ### Delete query by ID @@ -2375,9 +2336,6 @@ Deletes the query specified by ID. `Status: 200` -```json -{} -``` ### Delete queries @@ -2519,9 +2477,6 @@ Creates and/or modifies the queries included in the specs list. To modify an exi `Status: 200` -```json -{} -``` ### Live query health check @@ -2541,9 +2496,6 @@ None. `Status: 200` -```json -{} -``` ### live query result store health check @@ -2563,9 +2515,6 @@ None. `Status: 200` -```json -{} -``` ### Run live query @@ -3247,9 +3196,6 @@ None. `Status: 200` -```json -{} -``` --- @@ -3455,9 +3401,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json -{} -``` --- @@ -3691,9 +3634,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json -{} -``` ### Delete pack by ID @@ -3713,9 +3653,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json -{} -``` ### Get scheduled queries in a pack @@ -3965,9 +3902,6 @@ This allows you to easily configure scheduled queries that will impact a whole t `Status: 200` -```json -{} -``` ### Get packs specs @@ -4194,9 +4128,6 @@ Returns the specs for all packs in the Fleet instance. `Status: 200` -```json -{} -``` ### Get pack spec by name @@ -5071,7 +5002,7 @@ None. "enable_log_compression": false } } - } + }, "license": { "tier": "free", "organization": "fleet", @@ -5111,10 +5042,10 @@ None. "enable_log_compression": false } } - } + }, "update_interval": { "osquery_detail": 3600000000000 - } + }, } ``` @@ -5167,7 +5098,7 @@ Modifies the Fleet's configuration with the supplied information. "org_name": "Fleet Device Management", "org_logo_url": "https://fleetdm.com/logo.png" }, - "smtp_settings: { + "smtp_settings": { "enable_smtp": true, "server": "localhost", "port": "1025" @@ -5355,9 +5286,6 @@ Replaces the active global enroll secrets with the secrets specified. `Status: 200` -```json -{} -``` ### Get enroll secret for a team @@ -5410,17 +5338,17 @@ None. ```json { "email": "john_appleseed@example.com", - "name": John, + "name": "John", "sso_enabled": false, - "global_role": "admin" + "global_role": "admin", "teams": [ { "id": 2, - "role: "observer" + "role": "observer" }, { "id": 3, - "role: "maintainer" + "role": "maintainer" }, ] } @@ -5438,7 +5366,7 @@ None. "created_at": "0001-01-01T00:00:00Z", "updated_at": "0001-01-01T00:00:00Z", "id": 3, - "invited_by": 1 + "invited_by": 1, "email": "john_appleseed@example.com", "name": "John", "sso_enabled": false, @@ -5537,9 +5465,6 @@ Delete the specified invite from Fleet. `Status: 200` -```json -{} -``` ### Verify invite @@ -5633,7 +5558,7 @@ Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allo To initiate a file carve using the Fleet API, you can use the [live query](#run-live-query) or [scheduled query](#add-scheduled-query-to-a-pack) endpoints to run a query against the `carves` table. -For more information on executing a file carve in Fleet, go to the [File carving with Fleet docs](../1-Using-Fleet/2-fleetctl-CLI.md#file-carving-with-fleet). +For more information on executing a file carve in Fleet, go to the [File carving with Fleet docs](../01-Using-Fleet/02-fleetctl-CLI.md#file-carving-with-fleet). ### List carves @@ -5784,7 +5709,7 @@ _Available in Fleet Premium_ ```json { - "teams: [ + "teams": [ { "id": 1. "created_at": "2021-07-28T15:58:21Z", @@ -5892,10 +5817,10 @@ _Available in Fleet Premium_ ```json { - "teams: [ + "teams": [ { "name": "workstations", - "id": 1 + "id": 1, "user_ids": [], "host_ids": [], "user_count": 0, @@ -5963,7 +5888,7 @@ _Available in Fleet Premium_ { "team": { "name": "Workstations", - "id": 1 + "id": 1, "user_ids": [1, 17, 22, 32], "host_ids": [], "user_count": 4, @@ -6015,7 +5940,7 @@ _Available in Fleet Premium_ { "team": { "name": "Workstations", - "id": 1 + "id": 1, "user_ids": [1, 17, 22, 32], "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], "user_count": 4, @@ -6089,7 +6014,7 @@ _Available in Fleet Premium_ { "team": { "name": "Workstations", - "id": 1 + "id": 1, "user_ids": [1, 17, 22, 32], "host_ids": [3, 6, 7, 8, 9, 20, 32, 44], "user_count": 4, @@ -6141,9 +6066,6 @@ _Available in Fleet Premium_ `Status: 200` -```json -{} -``` --- diff --git a/website/.sailsrc b/website/.sailsrc index 9f58d730b0..7499bf8d2f 100644 --- a/website/.sailsrc +++ b/website/.sailsrc @@ -12,224 +12,264 @@ "url": "/docs", "title": "Readme.md", "lastModifiedAt": 1624049901000, - "htmlId": "docs--readme--9f534d32b2", - "meta": {} - }, - { - "url": "/docs/using-fleet/learn-how-to-use-fleet", - "title": "Learn how to use Fleet", - "lastModifiedAt": 1631573400000, - "htmlId": "docs--0-learn-how-to-use-f--1b80658ae8", - "meta": {} - }, - { - "url": "/docs/using-fleet/fleet-ui", - "title": "Fleet UI", - "lastModifiedAt": 1631640519000, - "htmlId": "docs--1-fleet-ui--ed954948be", - "meta": {} - }, - { - "url": "/docs/using-fleet/teams", - "title": "Teams", - "lastModifiedAt": 1629395421000, - "htmlId": "docs--10-teams--782f2af710", - "meta": {} - }, - { - "url": "/docs/using-fleet/usage-statistics", - "title": "Usage statistics", - "lastModifiedAt": 1624989594000, - "htmlId": "docs--11-usage-statistics--3ed9f3101b", - "meta": {} - }, - { - "url": "/docs/using-fleet/supported-browsers", - "title": "Supported browsers", - "lastModifiedAt": 1630452786000, - "htmlId": "docs--12-supported-browser--6f8b591603", - "meta": {} - }, - { - "url": "/docs/using-fleet/vulnerability-processing", - "title": "Vulnerability processing", - "lastModifiedAt": 1629761820000, - "htmlId": "docs--13-vulnerability-pro--edb754352c", - "meta": {} - }, - { - "url": "/docs/using-fleet/fleetctl-cli", - "title": "Fleetctl CLI", - "lastModifiedAt": 1631040955000, - "htmlId": "docs--2-fleetctl-cli--b4a4f6b08c", - "meta": {} - }, - { - "url": "/docs/using-fleet/rest-api", - "title": "REST API", - "lastModifiedAt": 1631555916000, - "htmlId": "docs--3-rest-api--0370e3eaff", - "meta": {} - }, - { - "url": "/docs/using-fleet/adding-hosts", - "title": "Adding hosts", - "lastModifiedAt": 1625588060000, - "htmlId": "docs--4-adding-hosts--f25bc11364", - "meta": {} - }, - { - "url": "/docs/using-fleet/osquery-logs", - "title": "Osquery logs", - "lastModifiedAt": 1624631015000, - "htmlId": "docs--5-osquery-logs--b2e649cc1f", - "meta": {} - }, - { - "url": "/docs/using-fleet/monitoring-fleet", - "title": "Monitoring Fleet", - "lastModifiedAt": 1630357234000, - "htmlId": "docs--6-monitoring-fleet--b1fa6e4a69", - "meta": {} - }, - { - "url": "/docs/using-fleet/security-best-practices", - "title": "Security best practices", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--7-security-best-prac--ad931bb00b", - "meta": {} - }, - { - "url": "/docs/using-fleet/updating-fleet", - "title": "Updating Fleet", - "lastModifiedAt": 1630641746000, - "htmlId": "docs--8-updating-fleet--1887128e93", - "meta": {} - }, - { - "url": "/docs/using-fleet/permissions", - "title": "Permissions", - "lastModifiedAt": 1630415095000, - "htmlId": "docs--9-permissions--905e9c08da", - "meta": {} - }, - { - "url": "/docs/using-fleet/faq", - "title": "FAQ", - "lastModifiedAt": 1627511403000, - "htmlId": "docs--faq--f96c7228ae", - "meta": {} - }, - { - "url": "/docs/using-fleet", - "title": "Using Fleet", - "lastModifiedAt": 1626938622000, - "htmlId": "docs--readme--b097d08746", + "htmlId": "docs--readme--27004f4448", + "sectionRelativeRepoPath": "README.md", "meta": {} }, { "url": "/docs/deploying/installation", "title": "Installation", - "lastModifiedAt": 1625173055000, - "htmlId": "docs--1-installation--fe7d4e2e74", - "meta": {} - }, - { - "url": "/docs/deploying/example-deployment-scenarios", - "title": "Example deployment scenarios", - "lastModifiedAt": 1625588060000, - "htmlId": "docs--3-example-deployment--b850738ae0", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--01-installation--c1f7b7262d", + "sectionRelativeRepoPath": "02-Deploying/01-Installation.md", "meta": {} }, { "url": "/docs/deploying/configuration", "title": "Configuration", - "lastModifiedAt": 1631134512000, - "htmlId": "docs--2-configuration--a242085fa7", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--02-configuration--25bb47a163", + "sectionRelativeRepoPath": "02-Deploying/02-Configuration.md", + "meta": {} + }, + { + "url": "/docs/deploying/example-deployment-scenarios", + "title": "Example deployment scenarios", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--03-example-deploymen--1d32b988ab", + "sectionRelativeRepoPath": "02-Deploying/03-Example-deployment-scenarios.md", "meta": {} }, { "url": "/docs/deploying/fleetctl-agent-updates", "title": "Fleetctl agent updates", - "lastModifiedAt": 1631165652000, - "htmlId": "docs--4-fleetctl-agent-upd--f6d6a601d4", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--04-fleetctl-agent-up--92c6890fa9", + "sectionRelativeRepoPath": "02-Deploying/04-fleetctl-agent-updates.md", "meta": {} }, { "url": "/docs/deploying/faq", "title": "FAQ", - "lastModifiedAt": 1627511403000, - "htmlId": "docs--faq--7abb678d36", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--faq--3ad91393ce", + "sectionRelativeRepoPath": "02-Deploying/FAQ.md", "meta": {} }, { "url": "/docs/deploying", "title": "Deploying", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--readme--a0a26f55e2", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--fb635b427f", + "sectionRelativeRepoPath": "02-Deploying/README.md", "meta": {} }, { "url": "/docs/contributing/building-fleet", "title": "Building Fleet", - "lastModifiedAt": 1629140343000, - "htmlId": "docs--1-building-fleet--a0d05ce171", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--01-building-fleet--abcea456d8", + "sectionRelativeRepoPath": "03-Contributing/01-Building-Fleet.md", "meta": {} }, { "url": "/docs/contributing/testing", "title": "Testing", - "lastModifiedAt": 1630685123000, - "htmlId": "docs--2-testing--20bd58879c", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--02-testing--2f307719a6", + "sectionRelativeRepoPath": "03-Contributing/02-Testing.md", "meta": {} }, { "url": "/docs/contributing/migrations", "title": "Migrations", - "lastModifiedAt": 1629743194000, - "htmlId": "docs--3-migrations--ee672f0676", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--03-migrations--b553b6254f", + "sectionRelativeRepoPath": "03-Contributing/03-Migrations.md", "meta": {} }, { "url": "/docs/contributing/committing-changes", "title": "Committing changes", - "lastModifiedAt": 1629140343000, - "htmlId": "docs--4-committing-changes--62d8075df1", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--04-committing-change--9b92fdc560", + "sectionRelativeRepoPath": "03-Contributing/04-Committing-Changes.md", "meta": {} }, { "url": "/docs/contributing/releasing-fleet", "title": "Releasing Fleet", - "lastModifiedAt": 1629929664000, - "htmlId": "docs--5-releasing-fleet--2b2a696ea0", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--05-releasing-fleet--1f39f77c64", + "sectionRelativeRepoPath": "03-Contributing/05-Releasing-Fleet.md", + "meta": {} + }, + { + "url": "/docs/contributing/seeding-data", + "title": "Seeding data", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--06-seeding-data--af5ac86a99", + "sectionRelativeRepoPath": "03-Contributing/06-Seeding-Data.md", "meta": {} }, { "url": "/docs/contributing/faq", "title": "FAQ", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--faq--92e0006bf2", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--faq--1b33e57806", + "sectionRelativeRepoPath": "03-Contributing/FAQ.md", "meta": {} }, { "url": "/docs/contributing", "title": "Contributing", - "lastModifiedAt": 1624893322000, - "htmlId": "docs--readme--d5e4f68946", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--6de1bc799d", + "sectionRelativeRepoPath": "03-Contributing/README.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/fleet-ui", + "title": "Fleet UI", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--01-fleet-ui--4b5755ee58", + "sectionRelativeRepoPath": "01-Using-Fleet/01-Fleet-UI.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/fleetctl-cli", + "title": "Fleetctl CLI", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--02-fleetctl-cli--2a521b49d6", + "sectionRelativeRepoPath": "01-Using-Fleet/02-fleetctl-CLI.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/rest-api", + "title": "REST API", + "lastModifiedAt": 1632174198000, + "htmlId": "docs--03-rest-api--f0b4e26bae", + "sectionRelativeRepoPath": "01-Using-Fleet/03-REST-API.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/adding-hosts", + "title": "Adding hosts", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--04-adding-hosts--9ffccb2221", + "sectionRelativeRepoPath": "01-Using-Fleet/04-Adding-hosts.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/osquery-logs", + "title": "Osquery logs", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--05-osquery-logs--7fbf2c5c5a", + "sectionRelativeRepoPath": "01-Using-Fleet/05-Osquery-logs.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/monitoring-fleet", + "title": "Monitoring Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--06-monitoring-fleet--83f7cca9f9", + "sectionRelativeRepoPath": "01-Using-Fleet/06-Monitoring-Fleet.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/security-best-practices", + "title": "Security best practices", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--07-security-best-pra--7ba1af6048", + "sectionRelativeRepoPath": "01-Using-Fleet/07-Security-best-practices.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/updating-fleet", + "title": "Updating Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--08-updating-fleet--3b4e821ee3", + "sectionRelativeRepoPath": "01-Using-Fleet/08-Updating-Fleet.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/permissions", + "title": "Permissions", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--09-permissions--eb9ac05ff5", + "sectionRelativeRepoPath": "01-Using-Fleet/09-Permissions.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/teams", + "title": "Teams", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--10-teams--bd0bdf9444", + "sectionRelativeRepoPath": "01-Using-Fleet/10-Teams.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/usage-statistics", + "title": "Usage statistics", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--11-usage-statistics--ccd73f532c", + "sectionRelativeRepoPath": "01-Using-Fleet/11-Usage-statistics.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/supported-browsers", + "title": "Supported browsers", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--12-supported-browser--c3a9c18d40", + "sectionRelativeRepoPath": "01-Using-Fleet/12-Supported-browsers.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/vulnerability-processing", + "title": "Vulnerability processing", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--13-vulnerability-pro--7a9b62b621", + "sectionRelativeRepoPath": "01-Using-Fleet/13-Vulnerability-Processing.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/faq", + "title": "FAQ", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--faq--75e099695e", + "sectionRelativeRepoPath": "01-Using-Fleet/FAQ.md", + "meta": {} + }, + { + "url": "/docs/using-fleet", + "title": "Using Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--0b226f5257", + "sectionRelativeRepoPath": "01-Using-Fleet/README.md", + "meta": {} + }, + { + "url": "/docs/using-fleet/learn-how-to-use-fleet", + "title": "Learn how to use Fleet", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--00-learn-how-to-use---95b515dfd1", + "sectionRelativeRepoPath": "01-Using-Fleet/00-Learn-how-to-use-Fleet.md", "meta": {} }, { "url": "/docs/using-fleet/configuration-files", "title": "Configuration files", - "lastModifiedAt": 1631296113000, - "htmlId": "docs--readme--7908cef8a3", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--dc5df431cb", + "sectionRelativeRepoPath": "01-Using-Fleet/configuration-files/README.md", "meta": {} }, { "url": "/docs/using-fleet/standard-query-library", "title": "Standard query library", - "lastModifiedAt": 1624049901000, - "htmlId": "docs--readme--d3c7d96146", + "lastModifiedAt": 1632163704000, + "htmlId": "docs--readme--db16aa6f37", + "sectionRelativeRepoPath": "01-Using-Fleet/standard-query-library/README.md", "meta": {} } ], @@ -264,11 +304,11 @@ "remediation": "N/A" }, { - "name": "Detect Linux hosts with high severity vulnerable versions of OpenSSL", + "name": "Get OpenSSL versions", "platforms": "Linux", "description": "Retrieves the OpenSSL version.", "query": "SELECT name AS name, version AS version, 'deb_packages' AS source FROM deb_packages WHERE name LIKE 'openssl%' UNION SELECT name AS name, version AS version, 'apt_sources' AS source FROM apt_sources WHERE name LIKE 'openssl%' UNION SELECT name AS name, version AS version, 'rpm_packages' AS source FROM rpm_packages WHERE name LIKE 'openssl%';", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Zach Wasserman", @@ -277,15 +317,15 @@ "htmlUrl": "https://github.com/zwass" } ], - "slug": "detect-linux-hosts-with-high-severity-vulnerable-versions-of-open-ssl", + "slug": "get-open-ssl-versions", "remediation": "N/A" }, { - "name": "Detect machines with Gatekeeper disabled", + "name": "Get whether Gatekeeper is disabled", "platforms": "macOS", "description": "Gatekeeper tries to ensure only trusted software is run on a mac machine.", "query": "SELECT * FROM gatekeeper WHERE assessments_enabled = 0;", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Zach Wasserman", @@ -294,16 +334,16 @@ "htmlUrl": "https://github.com/zwass" } ], - "slug": "detect-machines-with-gatekeeper-disabled", + "slug": "get-whether-gatekeeper-is-disabled", "remediation": "N/A" }, { - "name": "Detect presence of authorized SSH keys", + "name": "Get authorized SSH keys", "platforms": "macOS, Linux", "description": "Presence of authorized SSH keys may be unusual on laptops. Could be completely normal on servers, but may be worth auditing for unusual keys and/or changes.", "query": "SELECT username, authorized_keys. * FROM users CROSS JOIN authorized_keys USING (uid);", - "purpose": "Detection", - "remediation": "Check out the linked table (https://github.com/fleetdm/fleet/blob/32b4d53e7f1428ce43b0f9fa52838cbe7b413eed/handbook/queries/detect-hosts-with-high-severity-vulnerable-versions-of-openssl.md#table-of-vulnerable-openssl-versions) to determine if the installed version is a high severity vulnerability and view the corresponding CVE(s)", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Mike Thomas", @@ -312,7 +352,7 @@ "htmlUrl": "https://github.com/mike-j-thomas" } ], - "slug": "detect-presence-of-authorized-ssh-keys" + "slug": "get-authorized-ssh-keys" }, { "name": "Get authorized keys for Local Accounts", @@ -706,12 +746,12 @@ "remediation": "N/A" }, { - "name": "Detect unencrypted SSH keys for local accounts", + "name": "Get unencrypted SSH keys for local accounts", "platforms": "macOS, Linux, Windows, FreeBSD", "description": "Identify SSH keys created without a passphrase which can be used in Lateral Movement (MITRE. TA0008)", "query": "SELECT uid, username, description, path, encrypted FROM users CROSS JOIN user_ssh_keys using (uid) WHERE encrypted=0;", - "purpose": "Detection", - "remediation": "First, make the user aware about the impact of SSH keys. Then rotate the unencrypted keys detected.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -720,15 +760,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-unencrypted-ssh-keys-for-local-accounts" + "slug": "get-unencrypted-ssh-keys-for-local-accounts" }, { - "name": "Detect unencrypted SSH keys for domain joined accounts", + "name": "Get unencrypted SSH keys for domain joined accounts", "platforms": "macOS, Linux, Windows, FreeBSD", "description": "Identify SSH keys created without a passphrase which can be used in Lateral Movement (MITRE. TA0008)", "query": "SELECT uid, username, description, path, encrypted FROM users CROSS JOIN user_ssh_keys using (uid) WHERE encrypted=0 and username in (SELECT distinct(username) FROM last);", - "purpose": "Detection", - "remediation": "First, make the user aware about the impact of SSH keys. Then rotate the unencrypted keys detected.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -737,7 +777,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-unencrypted-ssh-keys-for-domain-joined-accounts" + "slug": "get-unencrypted-ssh-keys-for-domain-joined-accounts" }, { "name": "Get crontab jobs", @@ -774,12 +814,12 @@ "remediation": "N/A" }, { - "name": "Detect dynamic linker hijacking on Linux (MITRE. T1574.006)", + "name": "Get dynamic linker hijacking on Linux (MITRE. T1574.006)", "platforms": "Linux", "description": "Detect any processes that run with LD_PRELOAD environment variable", "query": "SELECT env.pid, env.key, env.value, p.name,p.path, p.cmdline, p.cwd FROM process_envs env join processes p USING (pid) WHERE key='LD_PRELOAD';", - "purpose": "Detection", - "remediation": "Identify the process/binary detected and confirm with the system's owner.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -788,15 +828,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-dynamic-linker-hijacking-on-linux-mitre-t-1574-006" + "slug": "get-dynamic-linker-hijacking-on-linux-mitre-t-1574-006" }, { - "name": "Detect dynamic linker hijacking on macOS (MITRE. T1574.006)", + "name": "Get dynamic linker hijacking on macOS (MITRE. T1574.006)", "platforms": "macOS", "description": "Detect any processes that run with DYLD_INSERT_LIBRARIES environment variable", "query": "SELECT env.pid, env.key, env.value, p.name,p.path, p.cmdline, p.cwd FROM process_envs env join processes p USING (pid) WHERE key='DYLD_INSERT_LIBRARIES';", - "purpose": "Detection", - "remediation": "Identify the process/binary detected and confirm with the system's owner.", + "purpose": "Informational", + "remediation": "N/A", "contributors": [ { "name": "Ahmed Elshaer", @@ -805,7 +845,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-dynamic-linker-hijacking-on-mac-os-mitre-t-1574-006" + "slug": "get-dynamic-linker-hijacking-on-mac-os-mitre-t-1574-006" }, { "name": "Get etc hosts entries", @@ -859,11 +899,11 @@ "remediation": "N/A" }, { - "name": "Detect active user accounts on servers", + "name": "Get active user accounts on servers", "platforms": "Linux", "description": "Domain Joined environment normally have root or other service account only and users are SSH-ing using their Domain Accounts.", "query": "SELECT * FROM shadow WHERE password_status='active' and username!='root';", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Ahmed Elshaer", @@ -872,15 +912,15 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-active-user-accounts-on-servers", + "slug": "get-active-user-accounts-on-servers", "remediation": "N/A" }, { - "name": "Detect Nmap scanner", + "name": "Get Nmap scanner", "platforms": "macOS, Linux, Windows, FreeBSD", - "description": "Detect Nmap scanner process, identify the user, parent, process details.", + "description": "Get Nmap scanner process, as well as its user, parent, and process details.", "query": "SELECT p.pid, name, p.path, cmdline, cwd, start_time, parent, (SELECT name FROM processes WHERE pid=p.parent) AS parent_name, (SELECT username FROM users WHERE uid=p.uid) AS username FROM processes as p WHERE cmdline like 'nmap%';", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": "Ahmed Elshaer", @@ -889,7 +929,7 @@ "htmlUrl": "https://github.com/anelshaer" } ], - "slug": "detect-nmap-scanner", + "slug": "get-nmap-scanner", "remediation": "N/A" }, { @@ -944,11 +984,11 @@ "remediation": "N/A" }, { - "name": "Detect Windows print spooler remote code execution vulnerability", + "name": "Get Windows print spooler remote code execution vulnerability", "platforms": "Windows", "description": "Detects devices that are potentially vulnerable to CVE-2021-1675 because the print spooler service is not disabled.", "query": "SELECT CASE cnt WHEN 2 THEN \"TRUE\" ELSE \"FALSE\" END \"Vulnerable\" FROM (SELECT name start_type, COUNT(name) AS cnt FROM services WHERE name = 'NTDS' or (name = 'Spooler' and start_type <> 'DISABLED')) WHERE cnt = 2;", - "purpose": "Detection", + "purpose": "Informational", "contributors": [ { "name": null, @@ -957,7 +997,7 @@ "htmlUrl": "https://github.com/maravedi" } ], - "slug": "detect-windows-print-spooler-remote-code-execution-vulnerability", + "slug": "get-windows-print-spooler-remote-code-execution-vulnerability", "remediation": "N/A" }, { @@ -978,7 +1018,7 @@ "remediation": "N/A" }, { - "name": "Find deleted files from disk", + "name": "Get processes that no longer exist on disk", "platforms": "Linux, macOS, Windows", "description": "Lists all processes of which the binary which launched them no longer exists on disk. Attackers often delete files from disk after launching process to mask presence.", "query": "SELECT name, path, pid FROM processes WHERE on_disk = 0;", @@ -991,11 +1031,96 @@ "htmlUrl": "https://github.com/alphabrevity" } ], - "slug": "find-deleted-files-from-disk", + "slug": "get-processes-that-no-longer-exist-on-disk", + "remediation": "N/A" + }, + { + "name": "Get user files matching a specific hash", + "platforms": "macOS, Linux", + "description": "Looks for specific hash in the Users/ directories for files that are less than 50MB (osquery file size limitation.)", + "query": "SELECT path,sha256 FROM hash WHERE path in (SELECT path FROM file WHERE size < 50000000 AND path LIKE \"\"/Users/%/Documents/%%\"\") AND sha256 = \"\"16d28cd1d78b823c4f961a6da78d67a8975d66cde68581798778ed1f98a56d75\"\";", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-user-files-matching-a-specific-hash", + "remediation": "N/A" + }, + { + "name": "Get local administrator accounts on macOS", + "platforms": "macOS", + "description": "The query allows you to check macOS systems for local administrator accounts.", + "query": "SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid = u.gid;", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-local-administrator-accounts-on-mac-os", + "remediation": "N/A" + }, + { + "name": "Get all listening ports, by process", + "platforms": "Linux, macOS, Windows", + "description": "List ports that are listening on all interfaces, along with the process to which they are attached.", + "query": "SELECT lp.address, lp.pid, lp.port, lp.protocol, p.name, p.path, p.cmdline FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.address = \"0.0.0.0\";", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-all-listening-ports-by-process", + "remediation": "N/A" + }, + { + "name": "Get whether TeamViewer is installed/running", + "platforms": "Windows", + "description": "Looks for the TeamViewer service running on machines. This is used often when attackers gain access to a machine, running TeamViewer to allow them to access a machine.", + "query": "SELECT display_name,status,s.pid,p.path FROM services AS s JOIN processes AS p USING(pid) WHERE s.name LIKE \"%teamviewer%\";", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-whether-team-viewer-is-installed-running", + "remediation": "N/A" + }, + { + "name": "Get malicious Python backdoors", + "platforms": "macOS, Linux, Windows", + "description": "Watches for the backdoored Python packages installed on system. See (http://www.nbu.gov.sk/skcsirt-sa-20170909-pypi/index.html)", + "query": "select case cnt when 0 then \"NONE_INSTALLED\" else \"INSTALLED\" end as \"Malicious Python Packages\",package_name,package_version from (select count(name) as cnt,nameas package_name,version as package_version,path as package_pathfrom python_packages where package_name in ('acqusition','apidev-coop','bzip','crypt','django-server','pwd','setup-tools','telnet','urlib3','urllib'));", + "purpose": "Informational", + "contributors": [ + { + "name": "AndrewB", + "handle": "alphabrevity", + "avatarUrl": "https://avatars.githubusercontent.com/u/3847973?v=4", + "htmlUrl": "https://github.com/alphabrevity" + } + ], + "slug": "get-malicious-python-backdoors", "remediation": "N/A" } ], - "queryLibraryYmlRepoPath": "docs/1-Using-Fleet/standard-query-library/standard-query-library.yml", + "queryLibraryYmlRepoPath": "docs/01-Using-Fleet/standard-query-library/standard-query-library.yml", "compiledPagePartialsAppPath": "views/partials/built-from-markdown" } } diff --git a/website/assets/styles/pages/docs/code-blocks.less b/website/assets/styles/pages/docs/code-blocks.less index 4a01c5a023..d7caa9932c 100644 --- a/website/assets/styles/pages/docs/code-blocks.less +++ b/website/assets/styles/pages/docs/code-blocks.less @@ -1,6 +1,6 @@ // lesshint-disable spaceAroundComma, trailingWhitespace - code:not(.nohighlight, .hljs) { + code:not(.hljs):not(.nohighlight) { background-color: @ui-off-white; border: 1px solid @border-lt-gray; color: @core-fleet-black-75; @@ -16,7 +16,7 @@ } - pre:not(.algolia-autocomplete, .ds-dropdown-menu) { + pre:not(.algolia-autocomplete):not(.ds-dropdown-menu):not(.json) { padding: 24px; border: 1px solid @border-lt-gray; border-radius: 6px; From b3bafee363e3cdafb6f83dd5fdac2cccf7f3789d Mon Sep 17 00:00:00 2001 From: Martavis Parker <47053705+martavis@users.noreply.github.com> Date: Tue, 21 Sep 2021 07:10:08 -0700 Subject: [PATCH 36/82] Team maintainer create & run new query, no save (#2141) * team maintainer create & run new query, no save * lint fixes * showing table sidebar for team maintainer * style fixes to no results * lint fix --- .../forms/queries/QueryForm/QueryForm.tsx | 50 +++++++++++++++++-- .../pages/queries/QueryPage/QueryPage.tsx | 8 ++- .../components/QueryResults/QueryResults.tsx | 1 - .../components/QueryResults/_styles.scss | 25 ++++++---- 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/frontend/components/forms/queries/QueryForm/QueryForm.tsx b/frontend/components/forms/queries/QueryForm/QueryForm.tsx index 03c8959dde..e23531d07e 100644 --- a/frontend/components/forms/queries/QueryForm/QueryForm.tsx +++ b/frontend/components/forms/queries/QueryForm/QueryForm.tsx @@ -40,9 +40,9 @@ interface IQueryFormProps { } interface IRenderProps { - nameText: string; - descText: string; queryValue: string; + nameText?: string; + descText?: string; queryError?: any; queryOnChange?: any; name?: IFormField; @@ -251,6 +251,42 @@ const QueryForm = ({ ); + const renderCreateForTeamMaintainer = ({ + queryValue, + queryOnChange, + queryError, + }: IRenderProps) => ( + <> +
    +

    New query

    + {baseError &&
    {baseError}
    } + + {renderLiveQueryWarning()} +
    + +
    + + + ); + const renderForGlobalAdminOrMaintainer = ({ nameText, descText, @@ -268,7 +304,7 @@ const QueryForm = ({ name?.onChange(evt.target.value) @@ -390,6 +426,14 @@ const QueryForm = ({ }); } + if (!isEditMode && isAnyTeamMaintainer) { + return renderCreateForTeamMaintainer({ + queryValue, + queryOnChange, + queryError, + }); + } + if (isAnyTeamMaintainer || isGlobalMaintainer) { return renderRunForMaintainer({ nameText, descText, queryValue }); } diff --git a/frontend/pages/queries/QueryPage/QueryPage.tsx b/frontend/pages/queries/QueryPage/QueryPage.tsx index 65ad2119f7..913d00b1b2 100644 --- a/frontend/pages/queries/QueryPage/QueryPage.tsx +++ b/frontend/pages/queries/QueryPage/QueryPage.tsx @@ -41,7 +41,9 @@ const QueryPage = ({ location: { query: URLQuerySearch }, }: IQueryPageProps) => { const queryIdForEdit = paramsQueryId ? parseInt(paramsQueryId, 10) : null; - const { isGlobalAdmin, isGlobalMaintainer } = useContext(AppContext); + const { isGlobalAdmin, isGlobalMaintainer, isAnyTeamMaintainer } = useContext( + AppContext + ); const { selectedOsqueryTable, setSelectedOsqueryTable } = useContext( QueryContext ); @@ -199,7 +201,9 @@ const QueryPage = ({ const isFirstStep = step === QUERIES_PAGE_STEPS[1]; const sidebarClass = isFirstStep && isSidebarOpen && "has-sidebar"; const showSidebar = - isFirstStep && isSidebarOpen && (isGlobalAdmin || isGlobalMaintainer); + isFirstStep && + isSidebarOpen && + (isGlobalAdmin || isGlobalMaintainer || isAnyTeamMaintainer); return (
    diff --git a/frontend/pages/queries/QueryPage/components/QueryResults/QueryResults.tsx b/frontend/pages/queries/QueryPage/components/QueryResults/QueryResults.tsx index 3ff6f6c662..cff8912b26 100644 --- a/frontend/pages/queries/QueryPage/components/QueryResults/QueryResults.tsx +++ b/frontend/pages/queries/QueryPage/components/QueryResults/QueryResults.tsx @@ -303,7 +303,6 @@ const QueryResults = ({ {isQueryFinished && hasNoResults ? (

    Your live query returned no results. -
    Expecting to see results? Check to see if the hosts you targeted reported “Online” or check out the diff --git a/frontend/pages/queries/QueryPage/components/QueryResults/_styles.scss b/frontend/pages/queries/QueryPage/components/QueryResults/_styles.scss index 4a2be63775..9ecf08e32a 100644 --- a/frontend/pages/queries/QueryPage/components/QueryResults/_styles.scss +++ b/frontend/pages/queries/QueryPage/components/QueryResults/_styles.scss @@ -122,6 +122,20 @@ font-weight: $bold; } } + &__tab-panel { + .no-results-message { + margin-top: $pad-xxlarge; + font-size: $small; + font-weight: $bold; + + span { + margin-top: $pad-medium; + font-size: $x-small; + font-weight: $regular; + display: block; + } + } + } } &__results-table-container, @@ -137,17 +151,6 @@ margin-top: $pad-large; box-sizing: border-box; overflow: auto; - - .kolide-spinner { - align-self: center; - } - - .no-results-message { - flex-grow: 1; - align-self: center; - text-align: center; - font-size: $x-small; - } } &__table { From 4650484960523254b9bd49f10dc0879456b0b733 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 21 Sep 2021 11:48:20 -0300 Subject: [PATCH 37/82] Remove fk label membership (#2157) * Remove fk from label membership * Add changes file * Fix tests * No need to IGNORE anymore --- changes/remove-fk-label-membership | 1 + cmd/fleet/serve.go | 4 ++ server/datastore/mysql/labels.go | 23 ++++--- server/datastore/mysql/labels_test.go | 66 +++++++++++++++++++ ...20155130_DropForeignKeysLabelMembership.go | 24 +++++++ server/datastore/mysql/schema.sql | 8 +-- server/fleet/datastore.go | 1 + server/mock/datastore_mock.go | 10 +++ 8 files changed, 120 insertions(+), 17 deletions(-) create mode 100644 changes/remove-fk-label-membership create mode 100644 server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go diff --git a/changes/remove-fk-label-membership b/changes/remove-fk-label-membership new file mode 100644 index 0000000000..d908e4f0f4 --- /dev/null +++ b/changes/remove-fk-label-membership @@ -0,0 +1 @@ +* Make label membership insertions less stressful for the database. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index d95beda2f3..ab90d2bba6 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -509,6 +509,10 @@ func cronCleanups(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, if err != nil { level.Error(logger).Log("err", "cleaning scheduled query stats", "details", err) } + err = ds.CleanupOrphanLabelMembership(ctx) + if err != nil { + level.Error(logger).Log("err", "cleaning label_membership", "details", err) + } err = trySendStatistics(ctx, ds, fleet.StatisticsFrequency, "https://fleetdm.com/api/v1/webhooks/receive-usage-analytics") if err != nil { diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index a006ffe193..c2fcbf65ab 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -370,14 +370,8 @@ func (d *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet. err := d.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // Complete inserts if necessary if len(vals) > 0 { - sql := ` - INSERT IGNORE INTO label_membership (updated_at, label_id, host_id) VALUES - ` - sql += strings.Join(bindvars, ",") + - ` - ON DUPLICATE KEY UPDATE - updated_at = VALUES(updated_at) - ` + sql := `INSERT INTO label_membership (updated_at, label_id, host_id) VALUES ` + sql += strings.Join(bindvars, ",") + ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` _, err := tx.ExecContext(ctx, sql, vals...) if err != nil { @@ -387,9 +381,7 @@ func (d *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet. // Complete deletions if necessary if len(removes) > 0 { - sql := ` - DELETE FROM label_membership WHERE host_id = ? AND label_id IN (?) - ` + sql := `DELETE FROM label_membership WHERE host_id = ? AND label_id IN (?)` query, args, err := sqlx.In(sql, host.ID, removes) if err != nil { return errors.Wrap(err, "IN for DELETE FROM label_membership") @@ -666,5 +658,12 @@ func (d *Datastore) LabelIDsByName(ctx context.Context, labels []string) ([]uint } return labelIDs, nil - +} + +func (d *Datastore) CleanupOrphanLabelMembership(ctx context.Context) error { + _, err := d.writer.ExecContext(ctx, `DELETE FROM label_membership where not exists (select 1 from labels where id=label_id) or not exists (select 1 from hosts where id=host_id)`) + if err != nil { + return errors.Wrap(err, "cleaning orphan label_membership by label") + } + return nil } diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index dabc903216..c5c01b38aa 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -57,6 +57,7 @@ func TestLabels(t *testing.T) { {"Save", testLabelsSave}, {"QueriesForCentOSHost", testLabelsQueriesForCentOSHost}, {"RecordNonExistentQueryLabelExecution", testLabelsRecordNonexistentQueryLabelExecution}, + {"LabelMembershipCleanup", testLabelMembershipCleanup}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -692,3 +693,68 @@ func testLabelsRecordNonexistentQueryLabelExecution(t *testing.T, db *Datastore) require.NoError(t, db.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{99999: ptr.Bool(true)}, time.Now())) } + +func testLabelMembershipCleanup(t *testing.T, ds *Datastore) { + setupTest := func() (*fleet.Host, *fleet.Label) { + host, err := ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + OsqueryHostID: "1", + }) + require.NoError(t, err) + require.NotNil(t, host) + + label := &fleet.Label{ + Name: "foo", + Query: "select * from foo;", + } + label, err = ds.NewLabel(context.Background(), label) + require.NoError(t, err) + + require.NoError(t, ds.RecordLabelQueryExecutions(context.Background(), host, map[uint]*bool{label.ID: ptr.Bool(true)}, time.Now())) + return host, label + } + + checkCount := func(before, after int) { + var count int + require.NoError(t, ds.writer.Get(&count, `SELECT count(*) FROM label_membership`)) + assert.Equal(t, before, count) + + require.NoError(t, ds.CleanupOrphanLabelMembership(context.Background())) + + require.NoError(t, ds.writer.Get(&count, `SELECT count(*) FROM label_membership`)) + assert.Equal(t, after, count) + } + + t.Run("none gone", func(t *testing.T) { + host, label := setupTest() + checkCount(1, 1) + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + require.NoError(t, ds.CleanupOrphanLabelMembership(context.Background())) + }) + t.Run("label gone", func(t *testing.T) { + host, label := setupTest() + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + checkCount(1, 0) + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + }) + t.Run("host gone", func(t *testing.T) { + host, label := setupTest() + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + checkCount(1, 0) + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + }) + t.Run("both gone", func(t *testing.T) { + host, label := setupTest() + require.NoError(t, ds.DeleteHost(context.Background(), host.ID)) + require.NoError(t, ds.DeleteLabel(context.Background(), label.Name)) + checkCount(1, 0) + }) +} diff --git a/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go b/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go new file mode 100644 index 0000000000..6c7466e722 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210920155130_DropForeignKeysLabelMembership.go @@ -0,0 +1,24 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210920155130, Down_20210920155130) +} + +func Up_20210920155130(tx *sql.Tx) error { + _, err := tx.Exec( + `ALTER TABLE label_membership DROP FOREIGN KEY fk_lm_host_id, DROP FOREIGN KEY fk_lm_label_id;`) + if err != nil { + return errors.Wrap(err, "dropping foreign keys for label_membership") + } + return nil +} + +func Down_20210920155130(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 2ffbec1364..d93fef6439 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -268,9 +268,7 @@ CREATE TABLE `label_membership` ( `label_id` int(10) unsigned NOT NULL, `host_id` int(10) unsigned NOT NULL, PRIMARY KEY (`host_id`,`label_id`), - KEY `idx_lm_label_id` (`label_id`), - CONSTRAINT `fk_lm_host_id` FOREIGN KEY (`host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `fk_lm_label_id` FOREIGN KEY (`label_id`) REFERENCES `labels` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + KEY `idx_lm_label_id` (`label_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; @@ -310,9 +308,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=103 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=104 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 0949db4d32..d3d3765a1f 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -287,6 +287,7 @@ type Datastore interface { DeleteScheduledQuery(ctx context.Context, id uint) error ScheduledQuery(ctx context.Context, id uint) (*ScheduledQuery, error) CleanupOrphanScheduledQueryStats(ctx context.Context) error + CleanupOrphanLabelMembership(ctx context.Context) error /////////////////////////////////////////////////////////////////////////////// // TeamStore diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 4c8218a9be..4bc590258f 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -221,6 +221,8 @@ type ScheduledQueryFunc func(ctx context.Context, id uint) (*fleet.ScheduledQuer type CleanupOrphanScheduledQueryStatsFunc func(ctx context.Context) error +type CleanupOrphanLabelMembershipFunc func(ctx context.Context) error + type NewTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) type SaveTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) @@ -601,6 +603,9 @@ type DataStore struct { CleanupOrphanScheduledQueryStatsFunc CleanupOrphanScheduledQueryStatsFunc CleanupOrphanScheduledQueryStatsFuncInvoked bool + CleanupOrphanLabelMembershipFunc CleanupOrphanLabelMembershipFunc + CleanupOrphanLabelMembershipFuncInvoked bool + NewTeamFunc NewTeamFunc NewTeamFuncInvoked bool @@ -1223,6 +1228,11 @@ func (s *DataStore) CleanupOrphanScheduledQueryStats(ctx context.Context) error return s.CleanupOrphanScheduledQueryStatsFunc(ctx) } +func (s *DataStore) CleanupOrphanLabelMembership(ctx context.Context) error { + s.CleanupOrphanLabelMembershipFuncInvoked = true + return s.CleanupOrphanLabelMembershipFunc(ctx) +} + func (s *DataStore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { s.NewTeamFuncInvoked = true return s.NewTeamFunc(ctx, team) From 1f324339f8ab6a92951a1208f97165bbf1054ead Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 21 Sep 2021 14:21:44 -0300 Subject: [PATCH 38/82] Add jitter to intervals (#2158) * Add max jitter percent config * Fix jitter calc * Remove comment * Reduce test jitter to make tests less flaky * Remove jitter entirely * Document new config * Fix doc link --- changes/add-jitter-percent | 1 + docs/02-Deploying/02-Configuration.md | 20 ++++++++++- server/config/config.go | 5 +++ server/datastore/mysql/labels.go | 33 ++--------------- server/datastore/mysql/labels_test.go | 48 +++++-------------------- server/fleet/datastore.go | 7 ++-- server/mock/datastore_mock.go | 6 ++-- server/service/service.go | 17 ++++++--- server/service/service_campaign_test.go | 2 +- server/service/service_osquery.go | 32 ++++++++++++----- server/service/service_osquery_test.go | 16 ++++----- 11 files changed, 88 insertions(+), 99 deletions(-) create mode 100644 changes/add-jitter-percent diff --git a/changes/add-jitter-percent b/changes/add-jitter-percent new file mode 100644 index 0000000000..7bc0e30d32 --- /dev/null +++ b/changes/add-jitter-percent @@ -0,0 +1 @@ +* Add jitter percent for osquery update intervals to prevent all hosts from returning data at roughly the same time. diff --git a/docs/02-Deploying/02-Configuration.md b/docs/02-Deploying/02-Configuration.md index 89e0568667..0b2f95938f 100644 --- a/docs/02-Deploying/02-Configuration.md +++ b/docs/02-Deploying/02-Configuration.md @@ -433,7 +433,7 @@ The address to serve the Fleet webserver. The TLS cert to use when terminating TLS. -See [TLS certificate considerations](./1-Installation.md#tls-certificate-considerations) for more information about certificates and Fleet. +See [TLS certificate considerations](./01-Installation.md#tls-certificate-considerations) for more information about certificates and Fleet. - Default value: `./tools/osquery/fleet.crt` - Environment variable: `FLEET_SERVER_CERT` @@ -727,6 +727,24 @@ Options are `filesystem`, `firehose`, `kinesis`, `lambda`, `pubsub`, and `stdout result_log_plugin: firehose ``` +###### osquery_max_jitter_percent + +Given an update interval (label, or details), this will add up to the defined percentage in randomness to the interval. + +The goal of this is to prevent all hosts from checking in with data at the same time. + +So for example, if the label_update_interval is 1h, and this is set to 10. It'll add up a random number between 0 and 6 minutes +to the amount of time it takes for fleet to give the host the label queries. + +- Default value: `10` +- Environment variable: `FLEET_OSQUERY_MAX_JITTER_PERCENT` +- Config file format: + + ``` + osquery: + max_jitter_percent: 10 + ``` + ##### Logging (Fleet server logging) ###### logging_debug diff --git a/server/config/config.go b/server/config/config.go index 1d7f977c84..5c666118a3 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -95,6 +95,7 @@ type OsqueryConfig struct { StatusLogFile string `yaml:"status_log_file"` ResultLogFile string `yaml:"result_log_file"` EnableLogRotation bool `yaml:"enable_log_rotation"` + MaxJitterPercent int `yaml:"max_jitter_percent"` } // LoggingConfig defines configs related to logging @@ -306,6 +307,8 @@ func (man Manager) addConfigs() { "(DEPRECATED: Use filesystem.result_log_file) Path for osqueryd result logs") man.addConfigBool("osquery.enable_log_rotation", false, "(DEPRECATED: Use filesystem.enable_log_rotation) Enable automatic rotation for osquery log files") + man.addConfigInt("osquery.max_jitter_percent", 10, + "Maximum percentage of the interval to add as jitter") // Logging man.addConfigBool("logging.debug", false, @@ -463,6 +466,7 @@ func (man Manager) LoadConfig() FleetConfig { LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"), DetailUpdateInterval: man.getConfigDuration("osquery.detail_update_interval"), EnableLogRotation: man.getConfigBool("osquery.enable_log_rotation"), + MaxJitterPercent: man.getConfigInt("osquery.max_jitter_percent"), }, Logging: LoggingConfig{ Debug: man.getConfigBool("logging.debug"), @@ -749,6 +753,7 @@ func TestConfig() FleetConfig { ResultLogPlugin: "filesystem", LabelUpdateInterval: 1 * time.Hour, DetailUpdateInterval: 1 * time.Hour, + MaxJitterPercent: 0, }, Logging: LoggingConfig{ Debug: true, diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index c2fcbf65ab..ca5482e033 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -283,39 +283,12 @@ func platformForHost(host *fleet.Host) string { return host.Platform } -func (d *Datastore) LabelQueriesForHost(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { +func (d *Datastore) LabelQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { var rows *sql.Rows var err error platform := platformForHost(host) - if host.LabelUpdatedAt.Before(cutoff) { - // Retrieve all labels (with matching platform) for this host - sql := ` - SELECT id, query - FROM labels - WHERE platform = ? OR platform = '' - AND label_membership_type = ? -` - rows, err = d.reader.QueryContext(ctx, sql, platform, fleet.LabelMembershipTypeDynamic) - } else { - // Retrieve all labels (with matching platform) iff there is a label - // that has been created since this host last reported label query - // executions - sql := ` - SELECT id, query - FROM labels - WHERE ((SELECT max(created_at) FROM labels WHERE platform = ? OR platform = '') > ?) - AND (platform = ? OR platform = '') - AND label_membership_type = ? -` - rows, err = d.reader.QueryContext( - ctx, - sql, - platform, - host.LabelUpdatedAt, - platform, - fleet.LabelMembershipTypeDynamic, - ) - } + query := `SELECT id, query FROM labels WHERE platform = ? OR platform = '' AND label_membership_type = ?` + rows, err = d.reader.QueryContext(ctx, query, platform, fleet.LabelMembershipTypeDynamic) if err != nil && err != sql.ErrNoRows { return nil, errors.Wrap(err, "selecting label queries for host") diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index c5c01b38aa..d1a7d77b71 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -80,10 +80,8 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { host.Platform = "darwin" require.NoError(t, db.SaveHost(context.Background(), host)) - baseTime := time.Now() - // No labels to check - queries, err := db.LabelQueriesForHost(context.Background(), host, baseTime) + queries, err := db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) assert.Len(t, queries, 0) @@ -127,7 +125,7 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { host.Platform = "darwin" // Now queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime) + queries, err = db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) assert.Equal(t, expectQueries, queries) @@ -136,6 +134,8 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { assert.Nil(t, err) assert.Len(t, labels, 1) + baseTime := time.Now() + // Record a query execution err = db.RecordLabelQueryExecutions( context.Background(), @@ -148,14 +148,6 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { require.NoError(t, err) host.LabelUpdatedAt = baseTime - // Now no queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) - assert.Nil(t, err) - assert.Len(t, queries, 0) - - // Ensure enough gap in created_at - time.Sleep(2 * time.Second) - // A new label targeting another platform should not effect the labels for // this host err = db.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{ @@ -166,9 +158,9 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { }, }) require.NoError(t, err) - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) + queries, err = db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) - assert.Len(t, queries, 0) + assert.Len(t, queries, 4) // If a new label is added, all labels should be returned err = db.ApplyLabelSpecs(context.Background(), []*fleet.LabelSpec{ @@ -180,29 +172,7 @@ func testLabelsAddAllHosts(t *testing.T, db *Datastore) { }) require.NoError(t, err) expectQueries["7"] = "query6" - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add(-1*time.Minute)) - assert.Nil(t, err) - assert.Len(t, queries, 5) - - // After expiration, all queries should be returned - queries, err = db.LabelQueriesForHost(context.Background(), host, baseTime.Add((2 * time.Minute))) - assert.Nil(t, err) - assert.Equal(t, expectQueries, queries) - - // Now the two matching labels should be returned - labels, err = db.ListLabelsForHost(context.Background(), host.ID) - assert.Nil(t, err) - if assert.Len(t, labels, 2) { - labelNames := []string{labels[0].Name, labels[1].Name} - sort.Strings(labelNames) - assert.Equal(t, "All Hosts", labelNames[0]) - assert.Equal(t, "label1", labelNames[1]) - } - - // A host that hasn't executed any label queries should still be asked - // to execute those queries - hosts[0].Platform = "darwin" - queries, err = db.LabelQueriesForHost(context.Background(), &hosts[0], time.Now()) + queries, err = db.LabelQueriesForHost(context.Background(), host) assert.Nil(t, err) assert.Len(t, queries, 5) @@ -663,9 +633,7 @@ func testLabelsQueriesForCentOSHost(t *testing.T, db *Datastore) { }) require.NoError(t, err) - baseTime := time.Now().Add(-5 * time.Minute) - - queries, err := db.LabelQueriesForHost(context.Background(), host, baseTime) + queries, err := db.LabelQueriesForHost(context.Background(), host) require.NoError(t, err) require.Len(t, queries, 1) assert.Equal(t, "select 1;", queries[fmt.Sprint(label.ID)]) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index d3d3765a1f..50b960b84a 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -139,10 +139,9 @@ type Datastore interface { Label(ctx context.Context, lid uint) (*Label, error) ListLabels(ctx context.Context, filter TeamFilter, opt ListOptions) ([]*Label, error) - // LabelQueriesForHost returns the label queries that should be executed for the given host. The cutoff is the - // minimum timestamp a query execution should have to be considered "fresh". Executions that are not fresh will be - // repeated. Results are returned in a map of label id -> query - LabelQueriesForHost(ctx context.Context, host *Host, cutoff time.Time) (map[string]string, error) + // LabelQueriesForHost returns the label queries that should be executed for the given host. + // Results are returned in a map of label id -> query + LabelQueriesForHost(ctx context.Context, host *Host) (map[string]string, error) // RecordLabelQueryExecutions saves the results of label queries. The results map is a map of label id -> whether or // not the label matches. The time parameter is the timestamp to save with the query execution. diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 4bc590258f..206d4433f4 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -115,7 +115,7 @@ type LabelFunc func(ctx context.Context, lid uint) (*fleet.Label, error) type ListLabelsFunc func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Label, error) -type LabelQueriesForHostFunc func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) +type LabelQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error) type RecordLabelQueryExecutionsFunc func(ctx context.Context, host *fleet.Host, results map[uint]*bool, t time.Time) error @@ -963,9 +963,9 @@ func (s *DataStore) ListLabels(ctx context.Context, filter fleet.TeamFilter, opt return s.ListLabelsFunc(ctx, filter, opt) } -func (s *DataStore) LabelQueriesForHost(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { +func (s *DataStore) LabelQueriesForHost(ctx context.Context, host *fleet.Host) (map[string]string, error) { s.LabelQueriesForHostFuncInvoked = true - return s.LabelQueriesForHostFunc(ctx, host, cutoff) + return s.LabelQueriesForHostFunc(ctx, host) } func (s *DataStore) RecordLabelQueryExecutions(ctx context.Context, host *fleet.Host, results map[uint]*bool, t time.Time) error { diff --git a/server/service/service.go b/server/service/service.go index e9cf216b89..62cec4c931 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -38,10 +38,19 @@ type Service struct { } // NewService creates a new service from the config struct -func NewService(ds fleet.Datastore, resultStore fleet.QueryResultStore, - logger kitlog.Logger, osqueryLogger *logging.OsqueryLogger, config config.FleetConfig, mailService fleet.MailService, - c clock.Clock, sso sso.SessionStore, lq fleet.LiveQueryStore, carveStore fleet.CarveStore, - license fleet.LicenseInfo) (fleet.Service, error) { +func NewService( + ds fleet.Datastore, + resultStore fleet.QueryResultStore, + logger kitlog.Logger, + osqueryLogger *logging.OsqueryLogger, + config config.FleetConfig, + mailService fleet.MailService, + c clock.Clock, + sso sso.SessionStore, + lq fleet.LiveQueryStore, + carveStore fleet.CarveStore, + license fleet.LicenseInfo, +) (fleet.Service, error) { var svc fleet.Service authorizer, err := authz.NewAuthorizer() diff --git a/server/service/service_campaign_test.go b/server/service/service_campaign_test.go index 6a6039cac5..5437cca55f 100644 --- a/server/service/service_campaign_test.go +++ b/server/service/service_campaign_test.go @@ -36,7 +36,7 @@ func TestStreamCampaignResultsClosesReditOnWSClose(t *testing.T) { campaign := &fleet.DistributedQueryCampaign{ID: 42} - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index fc5ab0bfdd..dce97d9332 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -2,8 +2,10 @@ package service import ( "context" + "crypto/rand" "encoding/json" "fmt" + "math/big" "strconv" "strings" "time" @@ -404,7 +406,7 @@ const hostDistributedQueryPrefix = "fleet_distributed_query_" // osqueryd to fill in the host details func (svc *Service) hostDetailQueries(ctx context.Context, host fleet.Host) (map[string]string, error) { queries := make(map[string]string) - if host.DetailUpdatedAt.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) && !host.RefetchRequested { + if !svc.shouldUpdate(host.DetailUpdatedAt, svc.config.Osquery.DetailUpdateInterval) && !host.RefetchRequested { // No need to update already fresh details return queries, nil } @@ -438,6 +440,19 @@ func (svc *Service) hostDetailQueries(ctx context.Context, host fleet.Host) (map return queries, nil } +func (svc *Service) shouldUpdate(lastUpdated time.Time, interval time.Duration) bool { + var jitter time.Duration + if svc.config.Osquery.MaxJitterPercent > 0 { + maxJitter := time.Duration(svc.config.Osquery.MaxJitterPercent) * interval / time.Duration(100.0) + randDuration, err := rand.Int(rand.Reader, big.NewInt(int64(maxJitter))) + if err == nil { + jitter = time.Duration(randDuration.Int64()) + } + } + cutoff := svc.clock.Now().Add(-(interval + jitter)) + return lastUpdated.Before(cutoff) +} + func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) { // skipauth: Authorization is currently for user endpoints only. svc.authz.SkipAuthorization(ctx) @@ -455,14 +470,15 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin } // Retrieve the label queries that should be updated - cutoff := svc.clock.Now().Add(-svc.config.Osquery.LabelUpdateInterval) - labelQueries, err := svc.ds.LabelQueriesForHost(ctx, &host, cutoff) - if err != nil { - return nil, 0, osqueryError{message: "retrieving label queries: " + err.Error()} - } + if svc.shouldUpdate(host.LabelUpdatedAt, svc.config.Osquery.LabelUpdateInterval) { + labelQueries, err := svc.ds.LabelQueriesForHost(ctx, &host) + if err != nil { + return nil, 0, osqueryError{message: "retrieving label queries: " + err.Error()} + } - for name, query := range labelQueries { - queries[hostLabelQueryPrefix+name] = query + for name, query := range labelQueries { + queries[hostLabelQueryPrefix+name] = query + } } liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID) diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 00570228cb..e2c6eb9ce0 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -324,7 +324,7 @@ func TestLabelQueries(t *testing.T) { Platform: "darwin", } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { @@ -361,7 +361,7 @@ func TestLabelQueries(t *testing.T) { assert.Len(t, queries, 0) assert.Zero(t, acc) - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{ "label1": "query1", "label2": "query2", @@ -539,7 +539,7 @@ func TestDetailQueriesWithEmptyStrings(t *testing.T) { ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil } - ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host, time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -715,7 +715,7 @@ func TestDetailQueries(t *testing.T) { ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil } - ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host, time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(context.Context, *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -920,7 +920,7 @@ func TestNewDistributedQueryCampaign(t *testing.T) { mockClock := clock.NewMockClock() svc := newTestServiceWithClock(ds, rs, lq, mockClock) - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { @@ -989,7 +989,7 @@ func TestDistributedQueryResults(t *testing.T) { campaign := &fleet.DistributedQueryCampaign{ID: 42} - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { @@ -1721,7 +1721,7 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) { }, nil } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { return nil } @@ -1813,7 +1813,7 @@ func TestPolicyQueries(t *testing.T) { Platform: "darwin", } - ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host, cutoff time.Time) (map[string]string, error) { + ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) { return map[string]string{}, nil } ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { From fd4c90eddf153e2cd2eff0d3825ae976455001f5 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Tue, 21 Sep 2021 14:19:19 -0400 Subject: [PATCH 39/82] terraform reference arch (#1761) * terraform initial architecture * added ecs autoscaling and https alb listener * add r53 hosted zone, dns cert verification, http -> https redirect * fleet dogfood env dogfood.fleetdm.com now configured, added license key, added readreplica settings, enabled vuln processing * add comment about using RDS serverless option --- .gitignore | 8 +- tools/terraform/.terraform-version | 1 + tools/terraform/ecs-iam.tf | 54 +++++ tools/terraform/ecs-sgs.tf | 78 +++++++ tools/terraform/ecs.tf | 328 +++++++++++++++++++++++++++++ tools/terraform/firehose.tf | 69 ++++++ tools/terraform/main.tf | 19 ++ tools/terraform/outputs.tf | 7 + tools/terraform/r53.tf | 93 ++++++++ tools/terraform/rds.tf | 113 ++++++++++ tools/terraform/readme.md | 44 ++++ tools/terraform/redis.tf | 65 ++++++ tools/terraform/variables.tf | 3 + tools/terraform/vpc.tf | 24 +++ 14 files changed, 905 insertions(+), 1 deletion(-) create mode 100644 tools/terraform/.terraform-version create mode 100644 tools/terraform/ecs-iam.tf create mode 100644 tools/terraform/ecs-sgs.tf create mode 100644 tools/terraform/ecs.tf create mode 100644 tools/terraform/firehose.tf create mode 100644 tools/terraform/main.tf create mode 100644 tools/terraform/outputs.tf create mode 100644 tools/terraform/r53.tf create mode 100644 tools/terraform/rds.tf create mode 100644 tools/terraform/readme.md create mode 100644 tools/terraform/redis.tf create mode 100644 tools/terraform/variables.tf create mode 100644 tools/terraform/vpc.tf diff --git a/.gitignore b/.gitignore index c1aa8fb7ba..b6049ee72b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,5 +50,11 @@ backup.sql.gz # committing a package-lock.json. Fleet app uses Yarn with yarn.lock. package-lock.json +# infra +.terraform +.terraform.tfstate* +.terraform.lock* +terraform.tfstate* + # generated installers -orbit-osquery* +orbit-osquery* \ No newline at end of file diff --git a/tools/terraform/.terraform-version b/tools/terraform/.terraform-version new file mode 100644 index 0000000000..a6a3a43c3a --- /dev/null +++ b/tools/terraform/.terraform-version @@ -0,0 +1 @@ +1.0.4 \ No newline at end of file diff --git a/tools/terraform/ecs-iam.tf b/tools/terraform/ecs-iam.tf new file mode 100644 index 0000000000..c726f85398 --- /dev/null +++ b/tools/terraform/ecs-iam.tf @@ -0,0 +1,54 @@ +data "aws_iam_policy_document" "fleet" { + statement { + effect = "Allow" + actions = ["cloudwatch:PutMetricData"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.database_password_secret.arn, data.aws_secretsmanager_secret.license.arn] + } + + statement { + effect = "Allow" + actions = [ + "firehose:DescribeDeliveryStream", + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + resources = [aws_kinesis_firehose_delivery_stream.osquery_logs.arn] + } +} + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "main" { + name = "fleetdm-role" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "role_attachment" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + role = aws_iam_role.main.name +} + +resource "aws_iam_policy" "main" { + name = "fleet-iam-policy" + policy = data.aws_iam_policy_document.fleet.json +} + +resource "aws_iam_role_policy_attachment" "attachment" { + policy_arn = aws_iam_policy.main.arn + role = aws_iam_role.main.name +} \ No newline at end of file diff --git a/tools/terraform/ecs-sgs.tf b/tools/terraform/ecs-sgs.tf new file mode 100644 index 0000000000..c5d3222dbd --- /dev/null +++ b/tools/terraform/ecs-sgs.tf @@ -0,0 +1,78 @@ +# Security group for the public internet facing load balancer +resource "aws_security_group" "lb" { + name = "${var.prefix} load balancer" + description = "${var.prefix} Load balancer security group" + vpc_id = module.vpc.vpc_id +} + +# Allow traffic from public internet +resource "aws_security_group_rule" "lb-ingress" { + description = "${var.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "443" + to_port = "443" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +resource "aws_security_group_rule" "lb-http-ingress" { + description = "${var.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "80" + to_port = "80" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +# Allow outbound traffic +resource "aws_security_group_rule" "lb-egress" { + description = "${var.prefix}: allow all outbound traffic" + type = "egress" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +# Security group for the backends that run the application. +# Allows traffic from the load balancer +resource "aws_security_group" "backend" { + name = "${var.prefix} backend" + description = "${var.prefix} Backend security group" + vpc_id = module.vpc.vpc_id + +} + +# Allow traffic from the load balancer to the backends +resource "aws_security_group_rule" "backend-ingress" { + description = "${var.prefix}: allow traffic from load balancer" + type = "ingress" + + from_port = "8080" + to_port = "8080" + protocol = "tcp" + source_security_group_id = aws_security_group.lb.id + security_group_id = aws_security_group.backend.id +} + +# Allow outbound traffic from the backends +resource "aws_security_group_rule" "backend-egress" { + description = "${var.prefix}: allow all outbound traffic" + type = "egress" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.backend.id +} \ No newline at end of file diff --git a/tools/terraform/ecs.tf b/tools/terraform/ecs.tf new file mode 100644 index 0000000000..090cc6a2da --- /dev/null +++ b/tools/terraform/ecs.tf @@ -0,0 +1,328 @@ +//resource "aws_route53_record" "record" { +// name = "fleetdm" +// type = "A" +// zone_id = "Z046188311R47QSK245X" +// alias { +// evaluate_target_health = false +// name = aws_alb.main.dns_name +// zone_id = aws_alb.main.zone_id +// } +//} + +resource "aws_alb" "main" { + name = "fleetdm" + internal = false + security_groups = [aws_security_group.lb.id, aws_security_group.backend.id] + subnets = module.vpc.public_subnets +} + +resource "aws_alb_target_group" "main" { + name = "fleetdm" + protocol = "HTTP" + target_type = "ip" + port = "8080" + vpc_id = module.vpc.vpc_id + deregistration_delay = 30 + + load_balancing_algorithm_type = "least_outstanding_requests" + + health_check { + path = "/healthz" + matcher = "200" + timeout = 10 + interval = 15 + healthy_threshold = 5 + unhealthy_threshold = 5 + } + + depends_on = [aws_alb.main] +} + +resource "aws_alb_listener" "https-fleetdm" { + load_balancer_arn = aws_alb.main.arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" + certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn + + default_action { + target_group_arn = aws_alb_target_group.main.arn + type = "forward" + } +} + +resource "aws_alb_listener" "http" { + load_balancer_arn = aws_alb.main.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_ecs_cluster" "fleet" { + name = "${var.prefix}-backend" + + setting { + name = "containerInsights" + value = "enabled" + } +} + +resource "aws_ecs_service" "fleet" { + name = "fleet" + launch_type = "FARGATE" + cluster = aws_ecs_cluster.fleet.id + task_definition = aws_ecs_task_definition.backend.arn + desired_count = 1 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + health_check_grace_period_seconds = 30 + + load_balancer { + target_group_arn = aws_alb_target_group.main.arn + container_name = "fleet" + container_port = 8080 + } + + network_configuration { + subnets = module.vpc.private_subnets + security_groups = [aws_security_group.backend.id] + } + + depends_on = [aws_alb_listener.http, aws_alb_listener.https-fleetdm] +} + +resource "aws_cloudwatch_log_group" "backend" { + name = "fleetdm" + retention_in_days = 1 +} + +data "aws_region" "current" {} + +data "aws_secretsmanager_secret" "license" { + name = "/fleet/license" +} + +resource "aws_ecs_task_definition" "backend" { + family = "fleet" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.main.arn + task_role_arn = aws_iam_role.main.arn + cpu = 512 + memory = 4096 + container_definitions = jsonencode( + [ + { + name = "fleet" + image = "fleetdm/fleet" + cpu = 512 + memory = 4096 + mountPoints = [] + volumesFrom = [] + essential = true + portMappings = [ + { + # This port is the same that the contained application also uses + containerPort = 8080 + protocol = "tcp" + } + ] + networkMode = "awsvpc" + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet" + } + }, + secrets = [ + { + name = "FLEET_MYSQL_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + }, + { + name = "FLEET_MYSQL_READ_REPLICA_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + }, + { + name = "FLEET_LICENSE_KEY" + valueFrom = data.aws_secretsmanager_secret.license.arn + } + ] + environment = [ + { + name = "FLEET_MYSQL_USERNAME" + value = "fleet" + }, + { + name = "FLEET_MYSQL_DATABASE" + value = "fleet" + }, + { + name = "FLEET_MYSQL_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_USERNAME" + value = "fleet" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_DATABASE" + value = "fleet" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_reader_endpoint}:3306" + }, + { + name = "FLEET_REDIS_ADDRESS" + value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" + }, + { + name = "FLEET_FIREHOSE_STATUS_STREAM" + value = aws_kinesis_firehose_delivery_stream.osquery_logs.name + }, + { + name = "FLEET_FIREHOSE_RESULT_STREAM" + value = aws_kinesis_firehose_delivery_stream.osquery_logs.name + }, + { + name = "FLEET_FIREHOSE_REGION" + value = data.aws_region.current.name + }, + { + name = "FLEET_OSQUERY_STATUS_LOG_PLUGIN" + value = "firehose" + }, + { + name = "FLEET_OSQUERY_RESULT_LOG_PLUGIN" + value = "firehose" + }, + { + name = "FLEET_SERVER_TLS" + value = "false" + }, + { + name = "FLEET_BETA_SOFTWARE_INVENTORY" + value = "1" + }, + { + name = "FLEET_VULNERABILITIES_DATABASES_PATH" + value = "/home/fleet" + } + ] + } + ]) +} + +resource "aws_ecs_task_definition" "migration" { + family = "fleet-migrate" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.main.arn + task_role_arn = aws_iam_role.main.arn + cpu = 256 + memory = 512 + container_definitions = jsonencode( + [ + { + name = "fleet-prepare-db" + image = "fleetdm/fleet" + cpu = 256 + memory = 512 + mountPoints = [] + volumesFrom = [] + essential = true + portMappings = [ + { + # This port is the same that the contained application also uses + containerPort = 8080 + protocol = "tcp" + } + ] + networkMode = "awsvpc" + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet" + } + }, + command = ["fleet", "prepare", "db"] + secrets = [ + { + name = "FLEET_MYSQL_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + } + ] + environment = [ + { + name = "FLEET_MYSQL_USERNAME" + value = "fleet" + }, + { + name = "FLEET_MYSQL_DATABASE" + value = "fleet" + }, + { + name = "FLEET_MYSQL_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" + }, + { + name = "FLEET_REDIS_ADDRESS" + value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" + } + ] + } + ]) +} + +resource "aws_appautoscaling_target" "ecs_target" { + max_capacity = 5 + min_capacity = 1 + resource_id = "service/${aws_ecs_cluster.fleet.name}/${aws_ecs_service.fleet.name}" + scalable_dimension = "ecs:service:DesiredCount" + service_namespace = "ecs" +} + +resource "aws_appautoscaling_policy" "ecs_policy_memory" { + name = "fleet-memory-autoscaling" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.ecs_target.resource_id + scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension + service_namespace = aws_appautoscaling_target.ecs_target.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageMemoryUtilization" + } + target_value = 80 + } +} + +resource "aws_appautoscaling_policy" "ecs_policy_cpu" { + name = "fleet-cpu-autoscaling" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.ecs_target.resource_id + scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension + service_namespace = aws_appautoscaling_target.ecs_target.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageCPUUtilization" + } + + target_value = 60 + } +} diff --git a/tools/terraform/firehose.tf b/tools/terraform/firehose.tf new file mode 100644 index 0000000000..ceed78ac42 --- /dev/null +++ b/tools/terraform/firehose.tf @@ -0,0 +1,69 @@ +resource "aws_s3_bucket" "osquery" { + bucket = "fleet-osquery-logs-archive" + acl = "private" + + lifecycle_rule { + enabled = true + expiration { + days = 1 + } + } + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + } + } + } +} + +// allow firehose to write to bucket +data "aws_iam_policy_document" "osquery_logs_policy_doc" { + statement { + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ] + resources = [aws_s3_bucket.osquery.arn, "${aws_s3_bucket.osquery.arn}/*"] + } +} + +resource "aws_iam_policy" "firehose" { + name = "osquery_logs_firehose_policy" + policy = data.aws_iam_policy_document.osquery_logs_policy_doc.json +} + +resource "aws_iam_role" "firehose" { + assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json +} + +resource "aws_iam_role_policy_attachment" "firehose" { + policy_arn = aws_iam_policy.firehose.arn + role = aws_iam_role.firehose.name +} + +data "aws_iam_policy_document" "osquery_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["firehose.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_kinesis_firehose_delivery_stream" "osquery_logs" { + name = "osquery_logs" + destination = "s3" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.osquery.arn + } +} \ No newline at end of file diff --git a/tools/terraform/main.tf b/tools/terraform/main.tf new file mode 100644 index 0000000000..95bb588b3e --- /dev/null +++ b/tools/terraform/main.tf @@ -0,0 +1,19 @@ +variable "region" { + default = "us-east-2" +} + +provider "aws" { + region = var.region +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "3.54.0" + } + } +} + +data "aws_caller_identity" "current" {} + diff --git a/tools/terraform/outputs.tf b/tools/terraform/outputs.tf new file mode 100644 index 0000000000..caa6ef2314 --- /dev/null +++ b/tools/terraform/outputs.tf @@ -0,0 +1,7 @@ +output "nameservers_fleetctl" { + value = aws_route53_zone.dogfood_fleetctl_com.name_servers +} + +output "nameservers_fleetdm" { + value = aws_route53_zone.dogfood_fleetdm_com.name_servers +} diff --git a/tools/terraform/r53.tf b/tools/terraform/r53.tf new file mode 100644 index 0000000000..619483a545 --- /dev/null +++ b/tools/terraform/r53.tf @@ -0,0 +1,93 @@ +resource "aws_route53_zone" "dogfood_fleetctl_com" { + name = "dogfood.fleetctl.com" +} + +resource "aws_route53_zone" "dogfood_fleetdm_com" { + name = "dogfood.fleetdm.com" +} + +resource "aws_route53_record" "dogfood_fleetctl_com" { + zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id + name = "dogfood.fleetctl.com" + type = "A" + + alias { + name = aws_alb.main.dns_name + zone_id = aws_alb.main.zone_id + evaluate_target_health = false + } +} + +resource "aws_route53_record" "dogfood_fleetdm_com" { + zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id + name = "dogfood.fleetdm.com" + type = "A" + + alias { + name = aws_alb.main.dns_name + zone_id = aws_alb.main.zone_id + evaluate_target_health = false + } +} + +resource "aws_acm_certificate" "dogfood_fleetctl_com" { + domain_name = "dogfood.fleetctl.com" + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_acm_certificate" "dogfood_fleetdm_com" { + domain_name = "dogfood.fleetdm.com" + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "dogfood_fleetctl_com_validation" { + for_each = { + for dvo in aws_acm_certificate.dogfood_fleetctl_com.domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id +} + +resource "aws_route53_record" "dogfood_fleetdm_com_validation" { + for_each = { + for dvo in aws_acm_certificate.dogfood_fleetdm_com.domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id +} + +resource "aws_acm_certificate_validation" "dogfood_fleetctl_com" { + certificate_arn = aws_acm_certificate.dogfood_fleetctl_com.arn + validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetctl_com_validation : record.fqdn] +} + +resource "aws_acm_certificate_validation" "dogfood_fleetdm_com" { + certificate_arn = aws_acm_certificate.dogfood_fleetdm_com.arn + validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetdm_com_validation : record.fqdn] +} \ No newline at end of file diff --git a/tools/terraform/rds.tf b/tools/terraform/rds.tf new file mode 100644 index 0000000000..c4a13466d6 --- /dev/null +++ b/tools/terraform/rds.tf @@ -0,0 +1,113 @@ +locals { + name = "fleetdm" +} + +resource "random_password" "database_password" { + length = 16 + special = false +} + +resource "aws_secretsmanager_secret" "database_password_secret" { + name = "/fleet/database/password/master" +} + +resource "aws_secretsmanager_secret_version" "database_password_secret_version" { + secret_id = aws_secretsmanager_secret.database_password_secret.id + secret_string = random_password.database_password.result +} + +// if you want to use RDS Serverless option prefer the following commented block +//module "aurora_mysql_serverless" { +// source = "terraform-aws-modules/rds-aurora/aws" +// version = "5.2.0" +// +// name = "${local.name}-mysql" +// engine = "aurora-mysql" +// engine_mode = "serverless" +// storage_encrypted = true +// username = "fleet" +// password = random_password.database_password.result +// create_random_password = false +// database_name = "fleet" +// enable_http_endpoint = true +// +// vpc_id = module.vpc.vpc_id +// subnets = module.vpc.database_subnets +// create_security_group = true +// allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks +// +// replica_scale_enabled = false +// replica_count = 0 +// +// monitoring_interval = 60 +// +// apply_immediately = true +// skip_final_snapshot = true +// +// db_parameter_group_name = aws_db_parameter_group.example_mysql.id +// db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id +// +// scaling_configuration = { +// auto_pause = true +// min_capacity = 2 +// max_capacity = 16 +// seconds_until_auto_pause = 300 +// timeout_action = "ForceApplyCapacityChange" +// } +//} + +module "aurora_mysql" { + source = "terraform-aws-modules/rds-aurora/aws" + version = "5.2.0" + + name = "${local.name}-mysql-iam" + engine = "aurora-mysql" + engine_version = "5.7.mysql_aurora.2.10.0" + instance_type = "db.t4g.medium" + instance_type_replica = "db.t4g.medium" + + iam_database_authentication_enabled = true + storage_encrypted = true + username = "fleet" + password = random_password.database_password.result + create_random_password = false + database_name = "fleet" + enable_http_endpoint = false + #performance_insights_enabled = true + + vpc_id = module.vpc.vpc_id + subnets = module.vpc.database_subnets + create_security_group = true + allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks + + replica_count = 1 + replica_scale_enabled = true + replica_scale_min = 1 + replica_scale_max = 3 + + monitoring_interval = 60 + iam_role_name = "${local.name}-rds-enhanced-monitoring" + iam_role_use_name_prefix = true + iam_role_description = "${local.name} RDS enhanced monitoring IAM role" + iam_role_path = "/autoscaling/" + iam_role_max_session_duration = 7200 + + apply_immediately = true + skip_final_snapshot = true + + db_parameter_group_name = aws_db_parameter_group.example_mysql.id + db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id + +} + +resource "aws_db_parameter_group" "example_mysql" { + name = "${local.name}-aurora-db-mysql-parameter-group" + family = "aurora-mysql5.7" + description = "${local.name}-aurora-db-mysql-parameter-group" +} + +resource "aws_rds_cluster_parameter_group" "example_mysql" { + name = "${local.name}-aurora-mysql-cluster-parameter-group" + family = "aurora-mysql5.7" + description = "${local.name}-aurora-mysql-cluster-parameter-group" +} \ No newline at end of file diff --git a/tools/terraform/readme.md b/tools/terraform/readme.md new file mode 100644 index 0000000000..5c6cb4baae --- /dev/null +++ b/tools/terraform/readme.md @@ -0,0 +1,44 @@ +## Terraform + +`terraform init && terraform workspace new dev` + +`terraform plan` + +`terraform apply` + +### Configuration + +Typical settings to override in an existing environment: + +`module.vpc.vpc_id` -- the VPC ID output from VPC module. If you are introducing fleet to an existing VPC, you could replace all instances with your VPC ID. + +In this reference architecture we are placing ECS, RDS MySQL, and Redis (ElastiCache) in separate subnets, each associated to a route table, allowing communication between. +This is not required, as long as Fleet can resolve the MySQL and Redis hosts, that should be adequate. + +#### HTTPS + +The ALB is in the public subnet with an ENI to bridge into the private subnet. SSL is terminated at the ALB and `fleet serve` is launched with `FLEET_SERVER_TLS=false` as an +environment variable. + +Replace `cert_arn` with the **certificate ARN** that applies to your environment. This is the **certificate ARN** used in the **ALB HTTPS Listener**. + +### Migrating the DB + +After applying terraform run the following to migrate the database: +``` +aws ecs run-task --cluster fleet-backend --task-definition fleet-migrate: --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets=[],securityGroups=[]}" +``` + +### Connecting a Host + +Build orbit: + +``` + fleetctl package --type=msi --fleet-url= --enroll-secret= +``` + +Run orbit: + +``` + "C:\Program Files\Orbit\bin\orbit\orbit.exe" --root-dir "C:\Program Files\Orbit\." --log-file "C:\Program Files\Orbit\orbit-log.txt" --fleet-url "http://" --enroll-secret-path "C:\Program Files\Orbit\secret.txt" --update-url "https://tuf.fleetctl.com" --orbit-channel "stable" --osqueryd-channel "stable" +``` \ No newline at end of file diff --git a/tools/terraform/redis.tf b/tools/terraform/redis.tf new file mode 100644 index 0000000000..d289472318 --- /dev/null +++ b/tools/terraform/redis.tf @@ -0,0 +1,65 @@ +variable "maintenance_window" { + default = "" +} +variable "engine_version" { + default = "5.0.6" +} +variable "node_type" { + default = "cache.t2.micro" +} +variable "number_cache_clusters" { + default = 3 +} +resource "aws_elasticache_replication_group" "default" { + availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"] + engine = "redis" + parameter_group_name = aws_elasticache_parameter_group.default.name + subnet_group_name = module.vpc.elasticache_subnet_group_name + security_group_ids = [aws_security_group.redis.id] + replication_group_id = "fleetdm-redis" + number_cache_clusters = var.number_cache_clusters + node_type = var.node_type + engine_version = var.engine_version + port = "6379" + maintenance_window = var.maintenance_window + snapshot_retention_limit = 0 + automatic_failover_enabled = false + at_rest_encryption_enabled = false + transit_encryption_enabled = false + apply_immediately = true + replication_group_description = "fleetdm-redis" +} + +resource "aws_elasticache_parameter_group" "default" { + name = "fleetdm-redis" + family = "redis5.0" + description = "for fleet" +} + +resource "aws_security_group" "redis" { + name = local.security_group_name + vpc_id = module.vpc.vpc_id +} + +locals { + security_group_name = "${var.prefix}-elasticache-redis" +} + +resource "aws_security_group_rule" "ingress" { + type = "ingress" + from_port = "6379" + to_port = "6379" + protocol = "tcp" + cidr_blocks = module.vpc.private_subnets_cidr_blocks + security_group_id = aws_security_group.redis.id +} + +resource "aws_security_group_rule" "egress" { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.redis.id +} + diff --git a/tools/terraform/variables.tf b/tools/terraform/variables.tf new file mode 100644 index 0000000000..40d60b7d4d --- /dev/null +++ b/tools/terraform/variables.tf @@ -0,0 +1,3 @@ +variable "prefix" { + default = "fleet" +} \ No newline at end of file diff --git a/tools/terraform/vpc.tf b/tools/terraform/vpc.tf new file mode 100644 index 0000000000..ed9fa8b4a7 --- /dev/null +++ b/tools/terraform/vpc.tf @@ -0,0 +1,24 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "fleet-vpc" + cidr = "10.10.0.0/16" + + azs = ["us-east-2a", "us-east-2b", "us-east-2c"] + private_subnets = ["10.10.1.0/24", "10.10.2.0/24", "10.10.3.0/24"] + public_subnets = ["10.10.11.0/24", "10.10.12.0/24", "10.10.13.0/24"] + database_subnets = ["10.10.21.0/24", "10.10.22.0/24", "10.10.23.0/24"] + elasticache_subnets = ["10.10.31.0/24", "10.10.32.0/24", "10.10.33.0/24"] + + create_database_subnet_group = true + create_database_subnet_route_table = true + + create_elasticache_subnet_group = true + create_elasticache_subnet_route_table = true + + enable_vpn_gateway = false + one_nat_gateway_per_az = false + + single_nat_gateway = true + enable_nat_gateway = true +} \ No newline at end of file From 0b7ff6cd38b123429bed4ef28e2dd0a32b24b7c4 Mon Sep 17 00:00:00 2001 From: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> Date: Wed, 22 Sep 2021 03:50:59 +0900 Subject: [PATCH 40/82] homepage "Try it out" text change (#2156) Updated the homepage call to action to read "Try it out". --- website/views/pages/homepage.ejs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/views/pages/homepage.ejs b/website/views/pages/homepage.ejs index b877c20cc6..d8be8c66ef 100644 --- a/website/views/pages/homepage.ejs +++ b/website/views/pages/homepage.ejs @@ -12,8 +12,8 @@

    Quickly deploy osquery and scale your fleet to 100,000+ devices on top of a stable core technology.
    -
    - Get started with Fleet +
    + Try it out Watch video @@ -136,7 +136,7 @@ class="btn btn-block btn-md btn-primary mx-4" href="/get-started" > - Get started + Try it out - Get started + Try it out Date: Tue, 21 Sep 2021 13:52:37 -0500 Subject: [PATCH 41/82] Fix broken link in markdown compilation (#2162) * handle links to fleetdm.com * Update build-static-content.js * Update build-static-content.js * update comment --- website/scripts/build-static-content.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/website/scripts/build-static-content.js b/website/scripts/build-static-content.js index 93e55dd189..f52a6044bf 100644 --- a/website/scripts/build-static-content.js +++ b/website/scripts/build-static-content.js @@ -226,13 +226,19 @@ module.exports = { // to some page on the destination site where this will be hosted, like `(*.)?fleetdm.com`. // If external, add target="_blank" so the link will open in a new tab. let isExternal = ! hrefString.match(/^href=\"https?:\/\/([^\.]+\.)*fleetdm\.com/g);// « FUTURE: make this smarter with sails.config.baseUrl + _.escapeRegExp() + // Check if this link is to fleetdm.com or www.fleetdm.com. + let isBaseUrl = hrefString.match(/^(href="https?:\/\/)([^\.]+\.)*fleetdm\.com"$/g); if (isExternal) { return hrefString.replace(/(href="https?:\/\/([^"]+)")/g, '$1 target="_blank"'); } else { // Otherwise, change the link to be web root relative. // (e.g. 'href="http://sailsjs.com/documentation/concepts"'' becomes simply 'href="/documentation/concepts"'') // > Note: See the Git version history of "compile-markdown-content.js" in the sailsjs.com website repo for examples of ways this can work across versioned subdomains. - return hrefString.replace(/href="https?:\/\//, '').replace(/^fleetdm\.com/, 'href="'); + if (isBaseUrl) { + return hrefString.replace(/href="https?:\/\//, '').replace(/([^\.]+\.)*fleetdm\.com/, 'href="/'); + } else { + return hrefString.replace(/href="https?:\/\//, '').replace(/^fleetdm\.com/, 'href="'); + } } });//∞ From f57b92ac2dfc43c4c5cbe6b0f27d67651ee0f0d9 Mon Sep 17 00:00:00 2001 From: William Theaker Date: Tue, 21 Sep 2021 14:55:38 -0400 Subject: [PATCH 42/82] Replace user permissions link with newer docs. (#2128) * Replace user permissions link with newer docs. * point at fleetdm.com/docs urls Co-authored-by: Mike McNeil --- .../admin/UserManagementPage/components/UserForm/UserForm.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/pages/admin/UserManagementPage/components/UserForm/UserForm.tsx b/frontend/pages/admin/UserManagementPage/components/UserForm/UserForm.tsx index ffd968bbdb..c7f6d8a44d 100644 --- a/frontend/pages/admin/UserManagementPage/components/UserForm/UserForm.tsx +++ b/frontend/pages/admin/UserManagementPage/components/UserForm/UserForm.tsx @@ -303,7 +303,7 @@ class UserForm extends Component { manage or observe all users, entities, and settings in Fleet.

    @@ -360,7 +360,7 @@ class UserForm extends Component { observe team-specific users, entities, and settings in Fleet.

    From 6497e0ba2e1ea79c82ec5b00a3f86bb606c7c479 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 21 Sep 2021 16:37:13 -0300 Subject: [PATCH 43/82] Improve performance of cascade host software migration (#2163) --- changes/improve-speed-of-migration | 1 + .../20210819131107_AddCascadeToHostSoftware.go | 18 ++++++++++++++---- server/datastore/mysql/migrations_test.go | 3 +++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 changes/improve-speed-of-migration diff --git a/changes/improve-speed-of-migration b/changes/improve-speed-of-migration new file mode 100644 index 0000000000..203c2faa3d --- /dev/null +++ b/changes/improve-speed-of-migration @@ -0,0 +1 @@ +* Improve the performance of certain database migrations that were preventing users from updating. diff --git a/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go b/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go index f1721cf57b..4de803112b 100644 --- a/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go +++ b/server/datastore/mysql/migrations/tables/20210819131107_AddCascadeToHostSoftware.go @@ -31,13 +31,13 @@ func Up_20210819131107(tx *sql.Tx) error { } // Clear any orphan software and host_software - _, err = tx.Exec(`DELETE FROM host_software WHERE NOT EXISTS (select 1 from hosts h where h.id=host_software.host_id)`) + _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_host_software AS SELECT * FROM host_software;`) if err != nil { - return errors.Wrap(err, "clearing orphan host_software") + return errors.Wrap(err, "save current host software to a temp table") } - _, err = tx.Exec(`DELETE FROM software WHERE NOT EXISTS (select 1 from host_software hs where hs.software_id=software.id)`) + _, err = tx.Exec(`DELETE FROM host_software;`) if err != nil { - return errors.Wrap(err, "clearing orphan software") + return errors.Wrap(err, "clear all host software") } if _, err := tx.Exec(` @@ -48,6 +48,16 @@ func Up_20210819131107(tx *sql.Tx) error { return errors.Wrap(err, "add fk on host_software hosts & software") } + _, err = tx.Exec(`INSERT IGNORE INTO host_software SELECT * FROM temp_host_software;`) + if err != nil { + return errors.Wrap(err, "reinserting host software") + } + + _, err = tx.Exec(`DROP TABLE temp_host_software;`) + if err != nil { + return errors.Wrap(err, "dropping temp table") + } + return nil } diff --git a/server/datastore/mysql/migrations_test.go b/server/datastore/mysql/migrations_test.go index f30833f41c..2398c5d8ac 100644 --- a/server/datastore/mysql/migrations_test.go +++ b/server/datastore/mysql/migrations_test.go @@ -111,7 +111,10 @@ func Test20210819131107_AddCascadeToHostSoftware(t *testing.T) { require.NoError(t, ds.DeleteHost(context.Background(), host1.ID)) + t.Log("Done adding software...") + startTime := time.Now() require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) + t.Log("took", time.Since(startTime)) // Make sure we don't delete more than we need hostCheck, err := ds.Host(context.Background(), host2.ID) From bd8cda15ce1674e5667a266ff4dc1cc41df2f854 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Tue, 21 Sep 2021 16:06:22 -0400 Subject: [PATCH 44/82] Publish development Docker images (#2114) Publish Docker images for the following events: 1) A user with write access to the repo opens a PR. 2) Any commit is made to the `main`, or `patch-*` branches. --- .../workflows/goreleaser-snapshot-fleet.yaml | 77 +++++++++++++++++++ .goreleaser-snapshot.yml | 74 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 .github/workflows/goreleaser-snapshot-fleet.yaml create mode 100644 .goreleaser-snapshot.yml diff --git a/.github/workflows/goreleaser-snapshot-fleet.yaml b/.github/workflows/goreleaser-snapshot-fleet.yaml new file mode 100644 index 0000000000..b2a255100e --- /dev/null +++ b/.github/workflows/goreleaser-snapshot-fleet.yaml @@ -0,0 +1,77 @@ +name: Docker publish + +on: + push: + branches: + - main + - patch-* + pull_request: + +jobs: + # This check-secrets job is used to gate execution of the main job. External PRs will not have + # access to the necessary secrets for pushing an image, so we want to skip that job if the secret + # is unavailable. + check-secrets: + environment: Docker Hub + runs-on: ubuntu-latest + outputs: + available: ${{ steps.check-secrets.outputs.available }} + steps: + - name: Check Secrets availability + id: check-secrets + run: | + if [ ! -z "${{ secrets.DOCKERHUB_USERNAME }}" ]; then + echo "::set-output name=available::true" + fi + + # If the secrets are available, build and publish a Docker image. + publish: + needs: + - check-secrets + if: ${{ needs.check-secrets.outputs.available == 'true' }} + runs-on: ubuntu-latest + environment: Docker Hub + steps: + - name: Checkout (PR) + if: ${{ github.event.pull_request.head.sha }} + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Checkout (main) + if: ${{ !github.event.pull_request.head.sha }} + uses: actions/checkout@v2 + + - name: Login to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # v1.10.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.5 + + - name: Install JS Dependencies + run: make deps-js + + - name: Install Go Dependencies + run: make deps-go + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@ac067437f516133269923265894e77920c3dce18 # v2.6.1 + with: + distribution: goreleaser-pro + version: latest + args: release --snapshot --rm-dist -f .goreleaser-snapshot.yml + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + + - name: Tag Docker image as branch + if: ${{ !github.event.pull_request.head.sha }} + run: docker tag fleetdm/fleet fleetdm/fleet:$(git rev-parse --abbrev-ref HEAD) + + # Explicitly push the docker images as GoReleaser will not do so in snapshot mode + - name: Publish Docker images + run: docker push fleetdm/fleet --all-tags diff --git a/.goreleaser-snapshot.yml b/.goreleaser-snapshot.yml new file mode 100644 index 0000000000..fa57b30145 --- /dev/null +++ b/.goreleaser-snapshot.yml @@ -0,0 +1,74 @@ +project_name: fleet + +monorepo: + tag_prefix: fleet- + dir: . + +before: + hooks: + - make deps + - make generate + +gomod: + proxy: true + +builds: + - id: fleet + dir: ./cmd/fleet/ + binary: fleet + env: + - CGO_ENABLED=1 + goos: + - linux + goarch: + - amd64 + flags: + - -tags=full,fts5,netgo + - -trimpath + ldflags: + - -extldflags "-static" + - -X github.com/kolide/kit/version.appName={{ .ArtifactName }} + - -X github.com/kolide/kit/version.version={{ .Version }} + - -X github.com/kolide/kit/version.branch={{ .Branch }} + - -X github.com/kolide/kit/version.revision={{ .FullCommit }} + - -X github.com/kolide/kit/version.buildDate={{ time "2006-01-02" }} + - -X github.com/kolide/kit/version.buildUser={{ .Env.USER }} + + - id: fleetctl + dir: ./cmd/fleetctl/ + binary: fleetctl + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + flags: + - -trimpath + ldflags: + - -X github.com/kolide/kit/version.appName={{ .ArtifactName }} + - -X github.com/kolide/kit/version.version={{ .Version }} + - -X github.com/kolide/kit/version.branch={{ .Branch }} + - -X github.com/kolide/kit/version.revision={{ .FullCommit }} + - -X github.com/kolide/kit/version.buildDate={{ time "2006-01-02" }} + - -X github.com/kolide/kit/version.buildUser={{ .Env.USER }} + + +dockers: + - goos: linux + goarch: amd64 + ids: + - fleet + - fleetctl + dockerfile: tools/docker/fleet.Dockerfile + image_templates: + - 'fleetdm/fleet:{{ .ShortCommit }}' + + - goos: linux + goarch: amd64 + ids: + - fleetctl + dockerfile: tools/docker/fleetctl.Dockerfile + image_templates: + - 'fleetdm/fleetctl:{{ .ShortCommit }}' + From e9c41038390ca4a8786cb4b6fb3aa8700eabb728 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Tue, 21 Sep 2021 15:54:34 -0500 Subject: [PATCH 45/82] Reduce noise in Slack (#2166) Only notify about zombie comments on issues or PRs that have been closed for >7 days (rather than >24 hours) --- website/api/controllers/webhooks/receive-from-github.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/api/controllers/webhooks/receive-from-github.js b/website/api/controllers/webhooks/receive-from-github.js index 190b899f88..b6dac23eee 100644 --- a/website/api/controllers/webhooks/receive-from-github.js +++ b/website/api/controllers/webhooks/receive-from-github.js @@ -220,7 +220,7 @@ module.exports = { (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS.includes(comment.user.login))|| (ghNoun === 'commit_comment' && ['created'].includes(action))|| (ghNoun === 'label' && ['created','edited','deleted'].includes(action) && GITHUB_USERNAME_OF_DRI_FOR_LABELS !== sender.login)||//« exempt label changes made by the directly responsible individual for labels, because otherwise when process changes/fiddlings happen, they can otherwise end up making too much noise in Slack - (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 24*60*60*1000 ) ) + (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 7*24*60*60*1000 ) ) ) { // ██╗███╗ ██╗███████╗ ██████╗ ██████╗ ███╗ ███╗ ██╗ ██╗███████╗ // ██║████╗ ██║██╔════╝██╔═══██╗██╔══██╗████╗ ████║ ██║ ██║██╔════╝ @@ -254,7 +254,7 @@ module.exports = { : (ghNoun === 'label') ? `@${sender.login} just ${action} a GitHub label "*${label.name}*" (#${label.color}) in ${repository.owner.login}/${repository.name}.\n\n> To manage labels in ${repository.owner.login}/${repository.name}, visit https://github.com/${encodeURIComponent(repository.owner.login)}/${encodeURIComponent(repository.name)}/labels` : - `@${sender.login} just created a zombie comment in a GitHub issue or PR that had already been closed for >24 hours (${issueOrPr.html_url}):\n\n> ${comment.html_url}\n\`\`\`\n${comment.body}\n\`\`\`` + `@${sender.login} just created a zombie comment in a GitHub issue or PR that had already been closed for >7 days (${issueOrPr.html_url}):\n\n> ${comment.html_url}\n\`\`\`\n${comment.body}\n\`\`\`` )+`\n` }, {'Content-Type':'application/json'} From 8b04b84b0cd980f18f133ee52d16466b07acad56 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Tue, 21 Sep 2021 14:01:38 -0700 Subject: [PATCH 46/82] Fix tagging branch name in development Docker publish (#2167) --- .github/workflows/goreleaser-snapshot-fleet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/goreleaser-snapshot-fleet.yaml b/.github/workflows/goreleaser-snapshot-fleet.yaml index b2a255100e..95ca8232b1 100644 --- a/.github/workflows/goreleaser-snapshot-fleet.yaml +++ b/.github/workflows/goreleaser-snapshot-fleet.yaml @@ -68,9 +68,9 @@ jobs: env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} - - name: Tag Docker image as branch + - name: Tag image with branch name if: ${{ !github.event.pull_request.head.sha }} - run: docker tag fleetdm/fleet fleetdm/fleet:$(git rev-parse --abbrev-ref HEAD) + run: docker tag fleetdm/fleet:$(git rev-parse --short HEAD) fleetdm/fleet:$(git rev-parse --abbrev-ref HEAD) # Explicitly push the docker images as GoReleaser will not do so in snapshot mode - name: Publish Docker images From accfa2218943b037e989f6ebc6c0b12880ede0f4 Mon Sep 17 00:00:00 2001 From: eashaw Date: Tue, 21 Sep 2021 16:17:45 -0500 Subject: [PATCH 47/82] updated styles, class names, and renderer (#2169) --- website/api/helpers/strings/to-html.js | 2 +- website/assets/styles/pages/docs/basic-documentation.less | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/website/api/helpers/strings/to-html.js b/website/api/helpers/strings/to-html.js index 0ec855ce7f..99f0dc8f96 100644 --- a/website/api/helpers/strings/to-html.js +++ b/website/api/helpers/strings/to-html.js @@ -77,7 +77,7 @@ module.exports = { var headingRenderer = new marked.Renderer(); headingRenderer.heading = function (text, level) { var headingID = _.kebabCase(text); - return ''+text+''; + return ''+text+'\n'; }; markedOpts.renderer = headingRenderer; } else { diff --git a/website/assets/styles/pages/docs/basic-documentation.less b/website/assets/styles/pages/docs/basic-documentation.less index b1394993c1..66a990dd03 100644 --- a/website/assets/styles/pages/docs/basic-documentation.less +++ b/website/assets/styles/pages/docs/basic-documentation.less @@ -38,9 +38,9 @@ } } - .docs-heading:hover { + .markdown-heading:hover { - .docs-link { + .markdown-link { height: 16px; vertical-align: middle; margin-left: 8px; @@ -352,7 +352,7 @@ padding-bottom: 24px; } - span + ul { + h1 + ul { display: none; // Hides links at top of some markdown files } From 8faea43990b9762059a918c7966fcf87c961ebd0 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Tue, 21 Sep 2021 18:43:27 -0500 Subject: [PATCH 48/82] fix fleetctl preview after standard query library yml moved (#2175) * fixes https://github.com/fleetdm/fleet/issues/2172 * also fixes contribute link on fleetdm.com/queries --- cmd/fleetctl/preview.go | 2 +- website/views/pages/query-library.ejs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/fleetctl/preview.go b/cmd/fleetctl/preview.go index 4ca9d3c1dd..c55750bca4 100644 --- a/cmd/fleetctl/preview.go +++ b/cmd/fleetctl/preview.go @@ -22,7 +22,7 @@ import ( const ( downloadUrl = "https://github.com/fleetdm/osquery-in-a-box/archive/master.zip" - standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/1-Using-Fleet/standard-query-library/standard-query-library.yml" + standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml" licenseKeyFlagName = "license-key" ) diff --git a/website/views/pages/query-library.ejs b/website/views/pages/query-library.ejs index 21d984c93b..06d83c3367 100644 --- a/website/views/pages/query-library.ejs +++ b/website/views/pages/query-library.ejs @@ -141,7 +141,7 @@

    Contributors

    Want to add your own query? Please submit a pull request - + over on GitHub .

    From d9b2f4a6fbe1c03aeafdfb598f8db481a5e6d9be Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Tue, 21 Sep 2021 17:08:58 -0700 Subject: [PATCH 49/82] Add --tag flag to fleetctl preview (#2171) Allows specifying a version of the Fleet image to run. --- changes/preview-tag | 1 + cmd/fleetctl/preview.go | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 changes/preview-tag diff --git a/changes/preview-tag b/changes/preview-tag new file mode 100644 index 0000000000..29f11331dd --- /dev/null +++ b/changes/preview-tag @@ -0,0 +1 @@ +* Allow specifying Fleet version in `fleetctl preview` with `--tag` flag. diff --git a/cmd/fleetctl/preview.go b/cmd/fleetctl/preview.go index c55750bca4..7afc3d9bf2 100644 --- a/cmd/fleetctl/preview.go +++ b/cmd/fleetctl/preview.go @@ -24,6 +24,7 @@ const ( downloadUrl = "https://github.com/fleetdm/osquery-in-a-box/archive/master.zip" standardQueryLibraryUrl = "https://raw.githubusercontent.com/fleetdm/fleet/main/docs/01-Using-Fleet/standard-query-library/standard-query-library.yml" licenseKeyFlagName = "license-key" + tagFlagName = "tag" ) func previewCommand() *cli.Command { @@ -45,6 +46,11 @@ Use the stop and reset subcommands to manage the server and dependencies once st Name: licenseKeyFlagName, Usage: "License key to enable Fleet Premium (optional)", }, + &cli.StringFlag{ + Name: tagFlagName, + Usage: "Run a specific version of Fleet", + Value: "latest", + }, }, Action: func(c *cli.Context) error { if err := checkDocker(); err != nil { @@ -72,6 +78,10 @@ Use the stop and reset subcommands to manage the server and dependencies once st return errors.Wrap(err, "make logs writable") } + if err := os.Setenv("FLEET_VERSION", c.String(tagFlagName)); err != nil { + return errors.Wrap(err, "failed to set Fleet version") + } + fmt.Println("Pulling Docker dependencies...") out, err := exec.Command("docker-compose", "pull").CombinedOutput() if err != nil { From 4d36400fe5b1aab56e37b0a6e6c096a32dd11efa Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Tue, 21 Sep 2021 18:23:11 -0700 Subject: [PATCH 50/82] Prepare for 4.3.1 release (#2177) --- CHANGELOG.md | 53 ++++++++++++++++--- changes/2107-sidebar-style | 1 - changes/2112-flaky-observer-hosts | 1 - changes/add-jitter-percent | 1 - changes/ensure-only-one-row-disk-space | 1 - changes/improve-speed-of-migration | 1 - changes/issue-1512-filter-queries | 1 - changes/issue-1893-team-policies | 1 - .../issue-1950-logging-filesystem-fail-early | 1 - changes/issue-1963-vulnerabilities-no-sync | 2 - changes/issue-1964-list-software | 1 - changes/issue-1969-redis-config | 2 - changes/issue-2062-team-maintainer-run-new | 1 - changes/preview-tag | 1 - changes/remove-fk-label-membership | 1 - changes/skip-save-users-if-disabled | 1 - charts/fleet/Chart.yaml | 4 +- charts/fleet/values.yaml | 2 +- .../deploy/terraform-aws-fargate/database.tf | 12 +++++ .../deploy/terraform-aws-fargate/variables.tf | 1 + tools/docker-fleetctl-awscli/Dockerfile | 4 ++ tools/fleetctl-npm/package.json | 2 +- 22 files changed, 67 insertions(+), 28 deletions(-) delete mode 100644 changes/2107-sidebar-style delete mode 100644 changes/2112-flaky-observer-hosts delete mode 100644 changes/add-jitter-percent delete mode 100644 changes/ensure-only-one-row-disk-space delete mode 100644 changes/improve-speed-of-migration delete mode 100644 changes/issue-1512-filter-queries delete mode 100644 changes/issue-1893-team-policies delete mode 100644 changes/issue-1950-logging-filesystem-fail-early delete mode 100644 changes/issue-1963-vulnerabilities-no-sync delete mode 100644 changes/issue-1964-list-software delete mode 100644 changes/issue-1969-redis-config delete mode 100644 changes/issue-2062-team-maintainer-run-new delete mode 100644 changes/preview-tag delete mode 100644 changes/remove-fk-label-membership delete mode 100644 changes/skip-save-users-if-disabled create mode 100644 tools/deploy/terraform-aws-fargate/database.tf create mode 100644 tools/deploy/terraform-aws-fargate/variables.tf create mode 100644 tools/docker-fleetctl-awscli/Dockerfile diff --git a/CHANGELOG.md b/CHANGELOG.md index 010e4736e1..8551a049a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +## Fleet 4.3.1 (Sept 21, 2021) + +* Add `fleetctl get software` to list all software and the detected vulnerabilities. + +* Add `fleetctl vulnerability-data-stream` command to sync the vulnerabilities processing data streams by hand. + +* Add `vulnerabilities.disable_data_sync` config to fleet serve to avoid downloading the data streams. + +* Allow specifying Fleet version in `fleetctl preview` with `--tag` flag. + +* Allow team maintainers to run new queries in the team hosts. + +* Only show observers queries they can run. + +* Add redis configuration option to retry failed connections. + +* Add redis configuration option to follow cluster redirections. + +* Add jitter percent for osquery update intervals to prevent all hosts from returning data at + roughly the same time. Note that this improves the Fleet server performance, but it will now take + longer for new labels to populate. + +* Improve the performance of certain database migrations that were preventing users from updating to + 4.3.0. + +* Reduce database load for label membership recording. + +* Add team policies. + +* Fix intermittent blank screen for observers on manage hosts page + +* Fix sidebar style on query page. + +* Fix a bug detecting disk space for hosts. + +* Fail early if the process does not have permissions to write to the logging file. + +* Completely skip trying to save host users and software inventory if it's disabled. + ## Fleet 4.3.0 (Sept 13, 2021) * Add Policies feature for detecting device compliance with organizational policies. @@ -138,7 +177,7 @@ * Add ability to create a Team schedule in Fleet. The Schedule feature was released in Fleet 4.1.0. For more information on the new Schedule feature, check out the [Fleet 4.1.0 release blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa25e89c1). *Available for Fleet Basic customers*. -* Add Beta Vulnerable software feature which surfaces vulnerable software on the **Host details** page and the `GET /api/v1/fleet/hosts/{id}` API route. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/13-Vulnerability-Processing.md#vulnerability-processing). +* Add Beta Vulnerable software feature which surfaces vulnerable software on the **Host details** page and the `GET /api/v1/fleet/hosts/{id}` API route. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/13-Vulnerability-Processing.md#vulnerability-processing). * Add ability to see which logging destination is configured for Fleet in the Fleet UI. To see this information, head to the **Schedule** page and then select "Schedule a query." Configured logging destination information is also available in the `GET api/v1/fleet/config` API route. @@ -148,9 +187,9 @@ * Add ability to modify scheduled queries in your Schedule in Fleet. The Schedule feature was released in Fleet 4.1.0. For more information on the new Schedule feature, check out the [Fleet 4.1.0 release blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa25e89c1). -* Add ability to disable the Users feature in Fleet by setting the new `enable_host_users` key to `true` in the `config` yaml, configuration file. For documentation on using configuration files in yaml syntax, check out the [Using yaml files in Fleet](https://github.com/fleetdm/fleet/tree/main/docs/1-Using-Fleet/configuration-files#using-yaml-files-in-fleet) documentation. +* Add ability to disable the Users feature in Fleet by setting the new `enable_host_users` key to `true` in the `config` yaml, configuration file. For documentation on using configuration files in yaml syntax, check out the [Using yaml files in Fleet](https://github.com/fleetdm/fleet/tree/main/docs/01-Using-Fleet/configuration-files#using-yaml-files-in-fleet) documentation. -* Improve performance of the Software inventory feature. Software inventory is currently under a feature flag. To enable this feature flag, check out the [feature flag documentation](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/2-Configuration.md#feature-flags). +* Improve performance of the Software inventory feature. Software inventory is currently under a feature flag. To enable this feature flag, check out the [feature flag documentation](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/02-Configuration.md#feature-flags). * Improve performance of inserting `pack_stats` in the database. The `pack_stats` information is used to display "Frequency" and "Last run" information for a specific host's scheduled queries. You can find this information on the **Host details** page. @@ -329,9 +368,9 @@ There are currently no known issues in this release. However, we recommend only The primary additions in Fleet 4.0.0 are the new Role-based access control (RBAC) and Teams features. -RBAC adds the ability to define a user's access to information and features in Fleet. This way, more individuals in an organization can utilize Fleet with appropriate levels of access. Check out the [permissions documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/9-Permissions.md) for a breakdown of the new user roles and their respective capabilities. +RBAC adds the ability to define a user's access to information and features in Fleet. This way, more individuals in an organization can utilize Fleet with appropriate levels of access. Check out the [permissions documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/09-Permissions.md) for a breakdown of the new user roles and their respective capabilities. -Teams adds the ability to separate hosts into exclusive groups. This way, users can easily observe and apply operations to consistent groups of hosts. Read more about the Teams feature in [the documentation here](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/10-Teams.md). +Teams adds the ability to separate hosts into exclusive groups. This way, users can easily observe and apply operations to consistent groups of hosts. Read more about the Teams feature in [the documentation here](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/10-Teams.md). There are several known issues that will be fixed for the stable release of Fleet 4.0.0. Therefore, we recommend only upgrading to Fleet 4.0.0 RC1 for testing purposes. Please file a GitHub issue for any issues discovered when testing Fleet 4.0.0! @@ -399,7 +438,7 @@ Fleet 4.0.0 is a major release and introduces several breaking changes and datab * Improve Fleet performance by batch updating host seen time instead of updating synchronously. This improvement reduces MySQL CPU usage by ~33% with 4,000 simulated hosts and MySQL running in Docker. -* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](./docs/2-Deploying/2-Configuration.md#software-inventory). +* Add support for software inventory, introducing a list of installed software items on each host's respective _Host details_ page. This feature is flagged off by default (for now). Check out [the feature flag documentation for instructions on how to turn this feature on](./docs/02-Deploying/02-Configuration.md#software-inventory). * Add Windows support for `fleetctl` agent autoupdates. The `fleetctl updates` command provides the ability to self-manage an agent update server. Available for Fleet Basic customers. @@ -867,7 +906,7 @@ to 2.0.0. ## Kolide Fleet 2.0.0 (currently preparing for release) -The primary new addition in Fleet 2 is the new `fleetctl` CLI and file-format, which dramatically increases the flexibility and control that administrators have over their osquery deployment. The CLI and the file format are documented [in the Fleet documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/2-fleetctl-CLI.md). +The primary new addition in Fleet 2 is the new `fleetctl` CLI and file-format, which dramatically increases the flexibility and control that administrators have over their osquery deployment. The CLI and the file format are documented [in the Fleet documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/02-fleetctl-CLI.md). ### New Features diff --git a/changes/2107-sidebar-style b/changes/2107-sidebar-style deleted file mode 100644 index 0a50d56a17..0000000000 --- a/changes/2107-sidebar-style +++ /dev/null @@ -1 +0,0 @@ -- Fixed sidebar style \ No newline at end of file diff --git a/changes/2112-flaky-observer-hosts b/changes/2112-flaky-observer-hosts deleted file mode 100644 index 8017ebafb2..0000000000 --- a/changes/2112-flaky-observer-hosts +++ /dev/null @@ -1 +0,0 @@ -- Fixed intermittent blank screen for observers on manage hosts page \ No newline at end of file diff --git a/changes/add-jitter-percent b/changes/add-jitter-percent deleted file mode 100644 index 7bc0e30d32..0000000000 --- a/changes/add-jitter-percent +++ /dev/null @@ -1 +0,0 @@ -* Add jitter percent for osquery update intervals to prevent all hosts from returning data at roughly the same time. diff --git a/changes/ensure-only-one-row-disk-space b/changes/ensure-only-one-row-disk-space deleted file mode 100644 index 1e23bec84e..0000000000 --- a/changes/ensure-only-one-row-disk-space +++ /dev/null @@ -1 +0,0 @@ -* Ensure only one row is returned when checking for disk space in hosts. diff --git a/changes/improve-speed-of-migration b/changes/improve-speed-of-migration deleted file mode 100644 index 203c2faa3d..0000000000 --- a/changes/improve-speed-of-migration +++ /dev/null @@ -1 +0,0 @@ -* Improve the performance of certain database migrations that were preventing users from updating. diff --git a/changes/issue-1512-filter-queries b/changes/issue-1512-filter-queries deleted file mode 100644 index 4ed072c52b..0000000000 --- a/changes/issue-1512-filter-queries +++ /dev/null @@ -1 +0,0 @@ -* Only show observers queries they can run. diff --git a/changes/issue-1893-team-policies b/changes/issue-1893-team-policies deleted file mode 100644 index 62e8b159ad..0000000000 --- a/changes/issue-1893-team-policies +++ /dev/null @@ -1 +0,0 @@ -* Add team policies. diff --git a/changes/issue-1950-logging-filesystem-fail-early b/changes/issue-1950-logging-filesystem-fail-early deleted file mode 100644 index 306f88436e..0000000000 --- a/changes/issue-1950-logging-filesystem-fail-early +++ /dev/null @@ -1 +0,0 @@ -* Fail early if the process does not have permissions to write to the logging file. diff --git a/changes/issue-1963-vulnerabilities-no-sync b/changes/issue-1963-vulnerabilities-no-sync deleted file mode 100644 index c99dd2d4a7..0000000000 --- a/changes/issue-1963-vulnerabilities-no-sync +++ /dev/null @@ -1,2 +0,0 @@ -* Add fleetctl vulnerability-data-stream command to sync the vulnerabilities processing data streams by hand. -* Add vulnerabilities.disable_data_sync config to fleet serve to avoid downloading the data streams. diff --git a/changes/issue-1964-list-software b/changes/issue-1964-list-software deleted file mode 100644 index c6607ea255..0000000000 --- a/changes/issue-1964-list-software +++ /dev/null @@ -1 +0,0 @@ -* Add `fleetctl get software` to list all software and the detected vulnerabilities. diff --git a/changes/issue-1969-redis-config b/changes/issue-1969-redis-config deleted file mode 100644 index dbbbcf214e..0000000000 --- a/changes/issue-1969-redis-config +++ /dev/null @@ -1,2 +0,0 @@ -* Add redis configuration option to retry failed connections. -* Add redis configuration option to follow cluster redirections. diff --git a/changes/issue-2062-team-maintainer-run-new b/changes/issue-2062-team-maintainer-run-new deleted file mode 100644 index eed0492a76..0000000000 --- a/changes/issue-2062-team-maintainer-run-new +++ /dev/null @@ -1 +0,0 @@ -* Allow team maintainers to run new queries in the team hosts. diff --git a/changes/preview-tag b/changes/preview-tag deleted file mode 100644 index 29f11331dd..0000000000 --- a/changes/preview-tag +++ /dev/null @@ -1 +0,0 @@ -* Allow specifying Fleet version in `fleetctl preview` with `--tag` flag. diff --git a/changes/remove-fk-label-membership b/changes/remove-fk-label-membership deleted file mode 100644 index d908e4f0f4..0000000000 --- a/changes/remove-fk-label-membership +++ /dev/null @@ -1 +0,0 @@ -* Make label membership insertions less stressful for the database. diff --git a/changes/skip-save-users-if-disabled b/changes/skip-save-users-if-disabled deleted file mode 100644 index 9c3f084555..0000000000 --- a/changes/skip-save-users-if-disabled +++ /dev/null @@ -1 +0,0 @@ -* Completely skip trying to save host users and software inventory if it's disabled. diff --git a/charts/fleet/Chart.yaml b/charts/fleet/Chart.yaml index 8f238ae4dd..66d0884e07 100644 --- a/charts/fleet/Chart.yaml +++ b/charts/fleet/Chart.yaml @@ -4,8 +4,8 @@ name: fleet keywords: - fleet - osquery -version: v4.3.0 +version: v4.3.1 home: https://github.com/fleetdm/fleet sources: - https://github.com/fleetdm/fleet.git -appVersion: v4.3.0 +appVersion: v4.3.1 diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 7dc1636ab2..3e9d3fdee8 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -2,7 +2,7 @@ # All settings related to how Fleet is deployed in Kubernetes hostName: fleet.localhost replicas: 3 # The number of Fleet instances to deploy -imageTag: v4.3.0 # Version of Fleet to deploy +imageTag: v4.3.1 # Version of Fleet to deploy createIngress: true # Whether or not to automatically create an Ingress ingressAnnotations: {} # Additional annotation to add to the Ingress podAnnotations: {} # Additional annotations to add to the Fleet pod diff --git a/tools/deploy/terraform-aws-fargate/database.tf b/tools/deploy/terraform-aws-fargate/database.tf new file mode 100644 index 0000000000..c87da3ade7 --- /dev/null +++ b/tools/deploy/terraform-aws-fargate/database.tf @@ -0,0 +1,12 @@ +resource "aws_db_instance" "default" { + allocated_storage = 10 + engine = "mysql" + engine_version = "5.7" + instance_class = "db.t3.micro" + identifier_prefix = "fleet" + name = "fleet" + username = "foo" + password = "foobarbaz" + parameter_group_name = "default.mysql5.7" + skip_final_snapshot = true +} \ No newline at end of file diff --git a/tools/deploy/terraform-aws-fargate/variables.tf b/tools/deploy/terraform-aws-fargate/variables.tf new file mode 100644 index 0000000000..b7bf843b5d --- /dev/null +++ b/tools/deploy/terraform-aws-fargate/variables.tf @@ -0,0 +1 @@ +variable "vpc_id" {} \ No newline at end of file diff --git a/tools/docker-fleetctl-awscli/Dockerfile b/tools/docker-fleetctl-awscli/Dockerfile new file mode 100644 index 0000000000..ff1fed9368 --- /dev/null +++ b/tools/docker-fleetctl-awscli/Dockerfile @@ -0,0 +1,4 @@ +FROM amazon/aws-cli +MAINTAINER Fleet Developers + +RUN curl https://github.com/fleetdm/fleet/releases/latest/download/fleetctl-linux.tar.gz | tar -xf diff --git a/tools/fleetctl-npm/package.json b/tools/fleetctl-npm/package.json index 3d03c517c5..8634b01ca6 100644 --- a/tools/fleetctl-npm/package.json +++ b/tools/fleetctl-npm/package.json @@ -1,6 +1,6 @@ { "name": "fleetctl", - "version": "v4.3.0", + "version": "v4.3.1", "description": "Installer for the fleetctl CLI tool", "bin": { "fleetctl": "./run.js" From 6413befdd182b2b64f8b5077246295a558fdd3bb Mon Sep 17 00:00:00 2001 From: eashaw Date: Wed, 22 Sep 2021 00:05:53 -0500 Subject: [PATCH 51/82] Add edit page button to the top of documentation pages on fleetdm.com/docs (#2165) * floating edit button on hover * update class name * adjust padding and border-radius * fix failing lint test * Update website/assets/styles/pages/docs/basic-documentation.less * Update website/assets/styles/pages/docs/basic-documentation.less * Update website/assets/styles/pages/docs/basic-documentation.less * Update website/assets/styles/pages/docs/basic-documentation.less Co-authored-by: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> --- .../pages/docs/basic-documentation.less | 50 +++++++++++++++++++ .../views/pages/docs/basic-documentation.ejs | 7 +++ 2 files changed, 57 insertions(+) diff --git a/website/assets/styles/pages/docs/basic-documentation.less b/website/assets/styles/pages/docs/basic-documentation.less index 66a990dd03..4d9956c5dd 100644 --- a/website/assets/styles/pages/docs/basic-documentation.less +++ b/website/assets/styles/pages/docs/basic-documentation.less @@ -198,6 +198,56 @@ color: @core-fleet-black; } + } + .edit-button-container { + position: relative; + + .edit-button { + color: @core-vibrant-blue; + opacity: 0; + font-size: 12px; + position: absolute; + right: 4px; + top: 8px; + cursor: pointer; + border: 1px solid @core-vibrant-blue; + border-radius: 4px; + padding: 4px 8px; + text-decoration: none; + + .edit-link { + color: @core-vibrant-blue; + + } + .edit-pencil { + height: 16px; + padding-right: 5px; + } + + } + + .edit-button:hover { + + background: @core-vibrant-blue; + + a { + text-decoration: none; + } + .edit-link { + color: @accent-white; + + } + } + + } + + &:hover { + + .edit-button { + opacity: 1; + + } + } [purpose='search'] { diff --git a/website/views/pages/docs/basic-documentation.ejs b/website/views/pages/docs/basic-documentation.ejs index 56c97246b3..7cc744d5d3 100644 --- a/website/views/pages/docs/basic-documentation.ejs +++ b/website/views/pages/docs/basic-documentation.ejs @@ -189,6 +189,13 @@
    + +
    +
    + Edit page +
    +
    + <%- partial( path.relative( path.dirname(__filename), From bc3d7fbe2b57cb8b6277aa786d6c4a07f0b970fa Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Wed, 22 Sep 2021 07:29:43 -0700 Subject: [PATCH 52/82] Always check doc links in CI (#2178) - Check all links on every PR to better avoid broken links. --- .github/workflows/docs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ed33f31b81..313c7e37d2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,7 +9,6 @@ jobs: - uses: actions/checkout@master - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: - check-modified-files-only: 'yes' use-quiet-mode: 'yes' config-file: .github/workflows/markdown-link-check-config.json base-branch: ${{ github.base_ref }} From 948e5ca943c23d2cbc12773a7ebed460995d9883 Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Wed, 22 Sep 2021 11:37:03 -0400 Subject: [PATCH 53/82] No clicking on new query (#2186) --- frontend/components/forms/queries/QueryForm/QueryForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/forms/queries/QueryForm/QueryForm.tsx b/frontend/components/forms/queries/QueryForm/QueryForm.tsx index e23531d07e..e6b28ec340 100644 --- a/frontend/components/forms/queries/QueryForm/QueryForm.tsx +++ b/frontend/components/forms/queries/QueryForm/QueryForm.tsx @@ -311,7 +311,7 @@ const QueryForm = ({ } /> ) : ( -

    New query

    +

    New query

    )} {isEditMode && ( Date: Wed, 22 Sep 2021 12:25:09 -0400 Subject: [PATCH 54/82] Update CHANGELOG entry for Fleet 4.3.1 (#2185) --- CHANGELOG.md | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8551a049a9..31bb2f8748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,41 +1,32 @@ ## Fleet 4.3.1 (Sept 21, 2021) -* Add `fleetctl get software` to list all software and the detected vulnerabilities. +* Add `fleetctl get software` command to list all software and the detected vulnerabilities. The Vulnerable software feature is currently in Beta. For information on how to configure the Vulnerable software feature and how exactly Fleet processes vulnerabilities, check out the [Vulnerability processing documentation](https://fleetdm.com/docs/using-fleet/vulnerability-processing). * Add `fleetctl vulnerability-data-stream` command to sync the vulnerabilities processing data streams by hand. -* Add `vulnerabilities.disable_data_sync` config to fleet serve to avoid downloading the data streams. +* Add `disable_data_sync` vulnerabilities configuration option to avoid downloading the data streams. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#disable-data-sync). -* Allow specifying Fleet version in `fleetctl preview` with `--tag` flag. +* Only show observers the queries they have permissions to run on the **Queries** page. In, Fleet 4.0.0, the Admin, Maintainer, and Observer user roles were introduced. Documentation for the permissions associated with each role can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/using-fleet/permissions). -* Allow team maintainers to run new queries in the team hosts. +* Add `connect_retry_attempts` Redis configuration option to retry failed connections. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#redis-connect-retry-attempts). -* Only show observers queries they can run. +* Add `cluster_follow_redirections` Redis configuration option to follow cluster redirections. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#redis-cluster-follow-redirections). -* Add redis configuration option to retry failed connections. +* Add `max_jitter_percent` osquery configuration option to prevent all hosts from returning data at roughly the same time. Note that this improves the Fleet server performance, but it will now take longer for new labels to populate. Documentation for this configuration option can be found [here on fleetdm.com/docs](https://fleetdm.com/docs/deploying/configuration#osquery-max-jitter-percent). -* Add redis configuration option to follow cluster redirections. - -* Add jitter percent for osquery update intervals to prevent all hosts from returning data at - roughly the same time. Note that this improves the Fleet server performance, but it will now take - longer for new labels to populate. - -* Improve the performance of certain database migrations that were preventing users from updating to - 4.3.0. +* Improve the performance of database migrations. * Reduce database load for label membership recording. -* Add team policies. - -* Fix intermittent blank screen for observers on manage hosts page - -* Fix sidebar style on query page. - -* Fix a bug detecting disk space for hosts. - * Fail early if the process does not have permissions to write to the logging file. -* Completely skip trying to save host users and software inventory if it's disabled. +* Completely skip trying to save a host's users and software inventory if it's disabled to reduce database load. + +* Fix a bug in which team maintainers were unable to run live queries against the hosts assigned to their team(s). + +* Fix a bug in which a blank screen would intermittently appear on the **Hosts** page. + +* Fix a bug detecting disk space for hosts. ## Fleet 4.3.0 (Sept 13, 2021) From ca27bd9d5c1b11eff474b4f8b0924f09636bda06 Mon Sep 17 00:00:00 2001 From: Renee Jackson <44620612+rlynnj11@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:28:25 -0300 Subject: [PATCH 55/82] fix broken links by adding missing 0 (#2187) * fix broken links by adding missing 0 * fix broken links take 2 gather links missed in first pass --- README.md | 2 +- cypress/README.md | 4 +-- docs/01-Using-Fleet/01-Fleet-UI.md | 2 +- docs/01-Using-Fleet/02-fleetctl-CLI.md | 4 +-- docs/01-Using-Fleet/04-Adding-hosts.md | 2 +- docs/01-Using-Fleet/05-Osquery-logs.md | 14 +++++----- docs/01-Using-Fleet/06-Monitoring-Fleet.md | 4 +-- .../07-Security-best-practices.md | 2 +- docs/01-Using-Fleet/08-Updating-Fleet.md | 4 +-- docs/01-Using-Fleet/10-Teams.md | 2 +- .../13-Vulnerability-Processing.md | 2 +- docs/01-Using-Fleet/FAQ.md | 28 +++++++++---------- docs/01-Using-Fleet/README.md | 16 +++++------ docs/02-Deploying/01-Installation.md | 10 +++---- .../03-Example-deployment-scenarios.md | 18 ++++++------ docs/02-Deploying/FAQ.md | 8 +++--- docs/02-Deploying/README.md | 8 +++--- docs/03-Contributing/02-Testing.md | 4 +-- docs/03-Contributing/04-Committing-Changes.md | 2 +- docs/03-Contributing/FAQ.md | 2 +- docs/03-Contributing/README.md | 10 +++---- docs/README.md | 6 ++-- frontend/README.md | 2 +- frontend/README_deprecated.md | 2 +- handbook/README.md | 2 +- handbook/manual-qa.md | 4 +-- handbook/release-process.md | 4 +-- handbook/support-process.md | 2 +- orbit/README.md | 2 +- tools/backup_db/README.md | 2 +- tools/fleetctl-npm/README.md | 2 +- 31 files changed, 88 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 6b622672a9..c3f8ce93e4 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The Fleet UI is now available at http://localhost:1337. #### Now what? -Check out the [Ask questions about your devices tutorial](./docs/1-Using-Fleet/tutorials/Ask-questions-about-your-devices.md) to learn where to see your devices in Fleet, how to add Fleet's standard query library, and how to ask questions about your devices by running queries. +Check out the [Ask questions about your devices tutorial](./docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md#how-to-ask-questions-about-your-devices) to learn where to see your devices in Fleet, how to add Fleet's standard query library, and how to ask questions about your devices by running queries. ## Team Fleet is [independently backed](https://linkedin.com/company/fleetdm) and actively maintained with the help of many amazing [contributors](https://github.com/fleetdm/fleet/graphs/contributors). diff --git a/cypress/README.md b/cypress/README.md index c67e59da62..ab80dd2069 100644 --- a/cypress/README.md +++ b/cypress/README.md @@ -1,6 +1,6 @@ # Cypress Testing -Cypress tests are designed solely for end-to-end testing. If this is your first time developing or running end-to-end tests, [Fleet testing documentation](../docs/3-Contributing/2-Testing.md) includes git instructions for test preparation and running tests. +Cypress tests are designed solely for end-to-end testing. If this is your first time developing or running end-to-end tests, [Fleet testing documentation](../docs/03-Contributing/02-Testing.md) includes git instructions for test preparation and running tests. ## Fleet Cypress directories @@ -37,6 +37,6 @@ As much as possible, assert that the code is only selecting 1 item or that the f ## Resources -- [Fleet testing documentation](../docs/3-Contributing/2-Testing.md) +- [Fleet testing documentation](../docs/03-Contributing/02-Testing.md) - [Cypress documentation](https://docs.cypress.io/api/table-of-contents) - [React testing-library query documentation](https://testing-library.com/docs/queries/about) diff --git a/docs/01-Using-Fleet/01-Fleet-UI.md b/docs/01-Using-Fleet/01-Fleet-UI.md index e6e0c68954..b90b2b2e89 100644 --- a/docs/01-Using-Fleet/01-Fleet-UI.md +++ b/docs/01-Using-Fleet/01-Fleet-UI.md @@ -27,7 +27,7 @@ To add queries to a pack, use the right-hand sidebar. You can take an existing s ![Schedule Query Sidebar](https://raw.githubusercontent.com/fleetdm/fleet/main/docs/images/schedule-query-sidebar.png) -Once you've scheduled queries and curated your packs, you can read our guide to [Working With Osquery Logs](../1-Using-Fleet/5-Osquery-logs.md). +Once you've scheduled queries and curated your packs, you can read our guide to [Working With Osquery Logs](../01-Using-Fleet/05-Osquery-logs.md). ## Configuring agent options diff --git a/docs/01-Using-Fleet/02-fleetctl-CLI.md b/docs/01-Using-Fleet/02-fleetctl-CLI.md index 396280f346..9e7ef885ae 100644 --- a/docs/01-Using-Fleet/02-fleetctl-CLI.md +++ b/docs/01-Using-Fleet/02-fleetctl-CLI.md @@ -24,7 +24,7 @@ This guide illustrates: ### Running Fleet -For the sake of this tutorial, we will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../3-Contributing/1-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): +For the sake of this tutorial, we will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../03-Contributing/01-Building-Fleet.md#development-infrastructure), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose): ``` docker-compose up -d @@ -186,7 +186,7 @@ spec: Fleet supports osquery's file carving functionality as of Fleet 3.3.0. This allows the Fleet server to request files (and sets of files) from osquery agents, returning the full contents to Fleet. -File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](../2-Deploying/2-Configuration.md#s3-file-carving-backend). +File carving data can be either stored in Fleet's database or to an external S3 bucket. For information on how to configure the latter, consult the [configuration docs](../02-Deploying/02-Configuration.md#s3-file-carving-backend). ### Configuration diff --git a/docs/01-Using-Fleet/04-Adding-hosts.md b/docs/01-Using-Fleet/04-Adding-hosts.md index 4872b7a204..9ea6e02e39 100644 --- a/docs/01-Using-Fleet/04-Adding-hosts.md +++ b/docs/01-Using-Fleet/04-Adding-hosts.md @@ -135,4 +135,4 @@ Multiple enroll secrets can be set to allow different groups of hosts to authenticate with Fleet. When a host enrolls, the corresponding enroll secret is recorded and can be used to segment hosts. -To set the enroll secret, use the `fleetctl` tool to apply an [enroll secret spec](../1-Using-Fleet/2-fleetctl-CLI.md#enroll-secrets) +To set the enroll secret, use the `fleetctl` tool to apply an [enroll secret spec](../01-Using-Fleet/02-fleetctl-CLI.md#enroll-secrets) diff --git a/docs/01-Using-Fleet/05-Osquery-logs.md b/docs/01-Using-Fleet/05-Osquery-logs.md index 96109f95c2..48223c68c6 100644 --- a/docs/01-Using-Fleet/05-Osquery-logs.md +++ b/docs/01-Using-Fleet/05-Osquery-logs.md @@ -22,21 +22,21 @@ Fleet supports the following logging plugins for osquery logs: - [PubSub](#pubsub) - Logs are written to Google Cloud PubSub topics. - [Stdout](#stdout) - Logs are written to stdout. -To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../2-Deploying/2-Configuration.md#options)). +To set the osquery logging plugins, use the `--osquery_result_log_plugin` and `--osquery_status_log_plugin` flags (or [equivalents for environment variables or configuration files](../02-Deploying/02-Configuration.md#options)). ### Filesystem The default logging plugin. - Plugin name: `filesystem` -- Flag namespace: [filesystem](../2-Deploying/2-Configuration.md#filesystem) +- Flag namespace: [filesystem](../02-Deploying/02-Configuration.md#filesystem) With the filesystem plugin, osquery result and/or status logs are written to the local filesystem on the Fleet server. This is typically used with a log forwarding agent on the Fleet server that will push the logs into a logging pipeline. Note that if multiple load-balanced Fleet servers are used, the logs will be load-balanced across those servers (not duplicated). ### Firehose - Plugin name: `firehose` -- Flag namespace: [firehose](../2-Deploying/2-Configuration.md#firehose) +- Flag namespace: [firehose](../02-Deploying/02-Configuration.md#firehose) With the Firehose plugin, osquery result and/or status logs are written to [AWS Firehose](https://aws.amazon.com/kinesis/data-firehose/) streams. This is a very good method for aggregating osquery logs into AWS S3 storage. @@ -45,7 +45,7 @@ Note that Firehose logging has limits [discussed in the documentation](https://d ### Kinesis - Plugin name: `kinesis` -- Flag namespace: [kinesis](../2-Deploying/2-Configuration.md#kinesis) +- Flag namespace: [kinesis](../02-Deploying/02-Configuration.md#kinesis) With the Kinesis plugin, osquery result and/or status logs are written to [AWS Kinesis](https://aws.amazon.com/kinesis/data-streams) streams. @@ -58,7 +58,7 @@ output in the Fleet logs and those logs _will not_ be sent to Kinesis. ### Lambda - Plugin name: `lambda` -- Flag namespace: [lambda](../2-Deploying/2-Configuration.md#lambda) +- Flag namespace: [lambda](../02-Deploying/02-Configuration.md#lambda) With the Lambda plugin, osquery result and/or status logs are written to [AWS Lambda](https://aws.amazon.com/lambda/) functions. @@ -79,7 +79,7 @@ Keep this in mind when using Lambda, as you're charged based on the number of re ### PubSub - Plugin name: `pubsub` -- Flag namespace: [pubsub](../2-Deploying/2-Configuration.md#pubsub) +- Flag namespace: [pubsub](../02-Deploying/02-Configuration.md#pubsub) With the PubSub plugin, osquery result and/or status logs are written to [PubSub](https://cloud.google.com/pubsub/) topics. @@ -88,7 +88,7 @@ Note that messages over 10MB will be dropped, with a notification sent to the fl ### Stdout - Plugin name: `stdout` -- Flag namespace: [stdout](../2-Deploying/2-Configuration.md#stdout) +- Flag namespace: [stdout](../02-Deploying/02-Configuration.md#stdout) With the stdout plugin, osquery result and/or status logs are written to stdout on the Fleet server. This is typically used for debugging or with a log diff --git a/docs/01-Using-Fleet/06-Monitoring-Fleet.md b/docs/01-Using-Fleet/06-Monitoring-Fleet.md index 99d1ac1ef2..391db88122 100644 --- a/docs/01-Using-Fleet/06-Monitoring-Fleet.md +++ b/docs/01-Using-Fleet/06-Monitoring-Fleet.md @@ -54,7 +54,7 @@ Scaling Fleet horizontally is as simple as running more Fleet server processes c The Fleet/osquery system is resilient to loss of availability. Osquery agents will continue executing the existing configuration and buffering result logs during downtime due to lack of network connectivity, server maintenance, or any other reason. Buffering in osquery can be configured with the `--buffered_log_max` flag. -Note that short downtimes are expected during [Fleet server upgrades](./8-Updating-Fleet.md)-fleet.md) that require database migrations. +Note that short downtimes are expected during [Fleet server upgrades](./08-Updating-Fleet.md)-fleet.md) that require database migrations. ### Debugging performance issues @@ -68,7 +68,7 @@ For performance issues in the Fleet server process, please [file an issue](https ##### Generate debug archive (Fleet 3.4.0+) -Use the `fleetctl archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./2-fleetctl-CLI.md)) for details on configuring `fleetctl`. +Use the `fleetctl archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./02-fleetctl-CLI.md)) for details on configuring `fleetctl`. The generated `.tar.gz` archive will be available in the current directory. diff --git a/docs/01-Using-Fleet/07-Security-best-practices.md b/docs/01-Using-Fleet/07-Security-best-practices.md index 4311907ce2..caa13d02b2 100644 --- a/docs/01-Using-Fleet/07-Security-best-practices.md +++ b/docs/01-Using-Fleet/07-Security-best-practices.md @@ -33,7 +33,7 @@ Passwords are never stored in plaintext in the database. We store a `bcrypt`ed h ### Authentication tokens -The size and expiration time of session tokens is admin-configurable. See [The documentation on session duration](../2-Deploying/2-Configuration.md#session_duration). +The size and expiration time of session tokens is admin-configurable. See [The documentation on session duration](../02-Deploying/02-Configuration.md#session_duration). It is possible to revoke all session tokens for a user by forcing a password reset. diff --git a/docs/01-Using-Fleet/08-Updating-Fleet.md b/docs/01-Using-Fleet/08-Updating-Fleet.md index af41724025..65d9c6e49c 100644 --- a/docs/01-Using-Fleet/08-Updating-Fleet.md +++ b/docs/01-Using-Fleet/08-Updating-Fleet.md @@ -7,7 +7,7 @@ ## Overview -This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../2-Deploying/1-Installation.md). +This guide explains how to update and run new versions of Fleet. For initial installation instructions, see [Installing Fleet](../02-Deploying/01-Installation.md). There are two steps to perform a typical Fleet update. If any other steps are required, they will be noted in the release notes. @@ -18,7 +18,7 @@ As with any enterprise software update, it's a good idea to back up your MySQL d ## Updating the Fleet binary -To update to a new version of Fleet, follow the [same binary install instructions](../2-Deploying/1-Installation.md) from the original installation method you used to install Fleet. +To update to a new version of Fleet, follow the [same binary install instructions](../02-Deploying/01-Installation.md) from the original installation method you used to install Fleet. ### Raw binaries diff --git a/docs/01-Using-Fleet/10-Teams.md b/docs/01-Using-Fleet/10-Teams.md index 86591f1055..4904e900db 100644 --- a/docs/01-Using-Fleet/10-Teams.md +++ b/docs/01-Using-Fleet/10-Teams.md @@ -94,7 +94,7 @@ To add users to a team: 4. Select one or more users by searching for their full name and confirm the action. -Users will be given the [Observer role](./9-Permissions.md#team-member-permissions) when added to the team. The [Edit a member's role](#edit-a-members-role) provides instructions on changing the permission level of users on a team. +Users will be given the [Observer role](./09-Permissions.md#team-member-permissions) when added to the team. The [Edit a member's role](#edit-a-members-role) provides instructions on changing the permission level of users on a team. ## Edit a member's role diff --git a/docs/01-Using-Fleet/13-Vulnerability-Processing.md b/docs/01-Using-Fleet/13-Vulnerability-Processing.md index 70092798c5..acb02e5a87 100644 --- a/docs/01-Using-Fleet/13-Vulnerability-Processing.md +++ b/docs/01-Using-Fleet/13-Vulnerability-Processing.md @@ -69,6 +69,6 @@ FLEET_VULNERABILITIES_DATABASES_PATH=/some/path The path specified needs to exist and Fleet needs to be able to read and write to and from it. This is the only mandatory configuration needed for vulnerability processing to work. Additional options, like vulnerability check frequency, can be -found in the [configuration documentation](../2-Deploying/2-Configuration.md#vulnerabilities). +found in the [configuration documentation](../02-Deploying/02-Configuration.md#vulnerabilities). You'll need to restart the Fleet instances after changing these settings. \ No newline at end of file diff --git a/docs/01-Using-Fleet/FAQ.md b/docs/01-Using-Fleet/FAQ.md index 99e0b69318..62771ea3e3 100644 --- a/docs/01-Using-Fleet/FAQ.md +++ b/docs/01-Using-Fleet/FAQ.md @@ -21,7 +21,7 @@ The upgrade from kolide/fleet to fleetdm/fleet works the same as any minor versi Minor version upgrades in Kolide Fleet often included database migrations and the recommendation to back up the database before migrating. The same goes for FleetDM Fleet versions. -To migrate from Kolide Fleet to FleetDM Fleet, please follow the steps outlined in the [Updating Fleet section](./8-Updating-Fleet.md) of the documentation. +To migrate from Kolide Fleet to FleetDM Fleet, please follow the steps outlined in the [Updating Fleet section](./08-Updating-Fleet.md) of the documentation. ## Has anyone stress tested Fleet? How many clients can the Fleet server handle? @@ -33,13 +33,13 @@ It’s standard deployment practice to have multiple Fleet servers behind a load No, currently, there’s no way to retrieve the name of the enroll secret with a query. This means that there's no way to create a label using your hosts' enroll secrets and then use this label as a target for queries or query packs. -Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./2-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. +Typically folks will use some other unique identifier to create labels that distinguish each type of device. As a workaround, [Fleet's manual labels](./02-fleetctl-CLI.md#host-labels) provide a way to create groups of hosts without a query. These manual labels can then be used as targets for queries or query packs. There is, however, a way to accomplish this even though the answer to the question remains "no": Teams. As of Fleet v4.0.0, you can group hosts in Teams either by enrolling them with a team specific secret, or by transferring hosts to a team. One the hosts you want to target are part of a team, you can create a query and target the team in question. ## How often do labels refresh? Is the refresh frequency configurable? -The update frequency for labels is configurable with the [—osquery_label_update_interval](../2-Deploying/2-Configuration.md#osquery_label_update_interval) flag (default 1 hour). +The update frequency for labels is configurable with the [—osquery_label_update_interval](../02-Deploying/02-Configuration.md#osquery_label_update_interval) flag (default 1 hour). ## How do I revoke the authorization tokens for a user? @@ -51,7 +51,7 @@ Fleet can live query the `osquery_schedule` table. Performing this live query al ## How do I monitor a Fleet server? -Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./6-Monitoring-Fleet.md) documentation for details. +Fleet provides standard interfaces for monitoring and alerting. See the [Monitoring Fleet](./06-Monitoring-Fleet.md) documentation for details. ## Why is the “Add User” button disabled? @@ -76,7 +76,7 @@ Live query results (executed in the web UI or `fleetctl query`) are pushed direc ### Scheduled Queries -Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../2-Deploying/2-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. +Scheduled query results (queries that are scheduled to run in Packs) are typically sent to the Fleet server, and will be available on the filesystem of the server at the path configurable by [`--osquery_result_log_file`](../02-Deploying/02-Configuration.md#osquery_result_log_file). This defaults to `/tmp/osquery_result`. It is possible to configure osqueryd to log query results outside of Fleet. For results to go to Fleet, the `--logger_plugin` flag must be set to `tls`. @@ -84,7 +84,7 @@ It is possible to configure osqueryd to log query results outside of Fleet. For Folks typically use Fleet to ship logs to data aggregation systems like Splunk, the ELK stack, and Graylog. -The [logger configuration options](../2-Deploying/2-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. +The [logger configuration options](../02-Deploying/02-Configuration.md#osquery_status_log_plugin) allow you to select the log output plugin. Using the log outputs you can route the logs to your chosen aggregation system. ### Troubleshooting @@ -94,7 +94,7 @@ Expecting results, but not seeing anything in the logs? - Check whether the query is scheduled in differential mode. If so, new results will only be logged when the result set changes. - Ensure that the query is scheduled to run on the intended platforms, and that the tables queried are supported by those platforms. - Use live query to `SELECT * FROM osquery_schedule` to check whether the query has been scheduled on the host. -- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../2-Deploying/2-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. +- Look at the status logs provided by osquery. In a standard configuration these are available on the filesystem of the Fleet server at the path configurable by [`--filesystem_status_log_file`](../02-Deploying/02-Configuration.md#filesystem_status_log_file). This defaults to `/tmp/osquery_status`. The host will output a status log each time it executes the query. ## Why aren’t my live queries being logged? @@ -104,17 +104,17 @@ Live query results are never logged to the filesystem of the Fleet server. See [ You cannot. Scheduled query results are logged to whatever logging plugin you have configured and are not stored in the Fleet DB. -However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./3-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./3-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). +However, the Fleet API exposes a significant amount of host information via the [`api/v1/fleet/hosts`](./03-REST-API.md#list-hosts) and the [`api/v1/fleet/hosts/{id}`](./03-REST-API.md#get-host) API endpoints. The `api/v1/fleet/hosts` [can even be configured to return additional host information](https://github.com/fleetdm/fleet/blob/9fb9da31f5462fa7dda4819a114bbdbc0252c347/docs/1-Using-Fleet/2-fleetctl-CLI.md#fleet-configuration-options). As an example, let's say you want to retrieve a host's OS version, installed software, and kernel version: -Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./3-REST-API.md#list-hosts). +Each host’s OS version is available using the `api/v1/fleet/hosts` API endpoint. [Check out the API documentation for this endpoint](./03-REST-API.md#list-hosts). -The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../2-Deploying/2-Configuration.md#feature-flags). +The ability to view each host’s installed software was released behind a feature flag in Fleet 3.11.0 and called Software inventory. [Check out the feature flag documentation for instructions on turning on Software inventory in Fleet](../02-Deploying/02-Configuration.md#feature-flags). -Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./3-REST-API.md#get-host). +Once the Software inventory feature is turned on, a list of a specific host’s installed software is available using the `api/v1/fleet/hosts/{id}` endpoint. [Check out the documentation for this endpoint](./03-REST-API.md#get-host). -It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./2-fleetctl-CLI.md#fleet-configuration-options). +It’s possible in Fleet to retrieve each host’s kernel version, using the Fleet API, through `additional_queries`. The Fleet configuration options yaml file includes an `additional_queries` property that allows you to append custom query results to the host details returned by the `api/v1/fleet/hosts` endpoint. [Check out an example configuration file with the additional_queries field](./02-fleetctl-CLI.md#fleet-configuration-options). ## How do I automatically add hosts to packs when the hosts enroll to Fleet? @@ -122,10 +122,10 @@ You can accomplish this by adding specific labels as targets of your pack. First When your hosts enroll to Fleet, they will become a member of the label and, because the label is a target of your pack, these hosts will automatically become targets of the pack. -You can also do this by setting the `targets` field in the [YAML configuration file](./2-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. +You can also do this by setting the `targets` field in the [YAML configuration file](./02-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. ## How do I resolve an "unknown column" error when upgrading Fleet? The `unknown column` error typically occurs when the database migrations haven't been run during the upgrade process. -Check out the [documentation on running database migrations](./8-Updating-Fleet.md#running-database-migrations) to resolve this issue. +Check out the [documentation on running database migrations](./08-Updating-Fleet.md#running-database-migrations) to resolve this issue. diff --git a/docs/01-Using-Fleet/README.md b/docs/01-Using-Fleet/README.md index 9dbd331a0b..1edfbc95e9 100644 --- a/docs/01-Using-Fleet/README.md +++ b/docs/01-Using-Fleet/README.md @@ -1,27 +1,27 @@ # Using Fleet -### [Fleet UI](./1-Fleet-UI.md) +### [Fleet UI](./01-Fleet-UI.md) Provides documentation about running and scheduling queries from within the Fleet UI -### [fleetctl CLI](./2-fleetctl-CLI.md) +### [fleetctl CLI](./02-fleetctl-CLI.md) Includes resources for setting up and configuring Fleet via the fleetctl CLI -### [REST API](./3-REST-API.md) +### [REST API](./03-REST-API.md) Provides resources for working with Fleet's API and includes example code for endpoints -### [Adding hosts](./4-Adding-hosts.md) +### [Adding hosts](./04-Adding-hosts.md) Provides resources for enrolling your hosts to Fleet -### [Osquery logs](./5-Osquery-logs.md) +### [Osquery logs](./05-Osquery-logs.md) Includes documentation on the plugin options for working with osquery logs -### [Monitoring Fleet](./6-Monitoring-Fleet.md) +### [Monitoring Fleet](./06-Monitoring-Fleet.md) Provides documentation for load balancer health checks and working with Fleet server metrics and performance -### [Security best practices](./7-Security-best-practices.md) +### [Security best practices](./07-Security-best-practices.md) Includes resources for ways to mitigate against the OWASP top 10 issues -### [Updating Fleet](./8-Updating-Fleet.md) +### [Updating Fleet](./08-Updating-Fleet.md) Includes a guide for how to update and run new versions of Fleet ### [FAQ](./FAQ.md) diff --git a/docs/02-Deploying/01-Installation.md b/docs/02-Deploying/01-Installation.md index 7a1f0f89c7..d59018f353 100644 --- a/docs/02-Deploying/01-Installation.md +++ b/docs/02-Deploying/01-Installation.md @@ -15,7 +15,7 @@ The Fleet application is distributed as a single static binary. This binary serv - The Fleet application API endpoints - The osquery TLS server API endpoints -All of these are served via a built-in HTTP server, so there is no need for complex web server configurations. Once you've installed the `fleet` binary and it's infrastructure dependencies as illustrated below, refer to the [Configuration](./2-Configuration.md) documentation for information on how to use and configure the Fleet application. +All of these are served via a built-in HTTP server, so there is no need for complex web server configurations. Once you've installed the `fleet` binary and it's infrastructure dependencies as illustrated below, refer to the [Configuration](./02-Configuration.md) documentation for information on how to use and configure the Fleet application. ## Installing the Fleet binary @@ -29,7 +29,7 @@ Pull the latest Fleet docker image: docker pull fleetdm/fleet ``` -For more information on using Fleet, refer to the [Configuration](./2-Configuration.md) documentation. +For more information on using Fleet, refer to the [Configuration](./02-Configuration.md) documentation. ### Raw binaries @@ -47,7 +47,7 @@ unzip fleet.zip 'linux/*' -d fleet ./fleet/linux/fleet_linux_amd64 --help ``` -For more information on using Fleet, refer to the [Configuration](./2-Configuration.md) documentation. +For more information on using Fleet, refer to the [Configuration](./02-Configuration.md) documentation. ## TLS configuration @@ -65,7 +65,7 @@ Fleet currently has two infrastructure dependencies in addition to the `fleet` w ### MySQL -Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well supported MySQL [Docker container](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](./2-Configuration.md) document. +Fleet uses MySQL extensively as its main database. Many cloud providers (such as [AWS](https://aws.amazon.com/rds/mysql/) and [GCP](https://cloud.google.com/sql/)) host reliable MySQL services which you may consider for this purpose. A well supported MySQL [Docker container](https://hub.docker.com/_/mysql/) also exists if you would rather run MySQL in a container. For more information on how to configure the `fleet` binary to use the correct MySQL instance, see the [Configuration](./02-Configuration.md) document. Fleet requires at least MySQL version 5.7. @@ -73,4 +73,4 @@ For host expiry configuration, the [event scheduler](https://dev.mysql.com/doc/r ### Redis -Fleet uses Redis to ingest and queue the results of distributed queries, cache data, etc. Many cloud providers (such as [AWS](https://aws.amazon.com/elasticache/) and [GCP](https://console.cloud.google.com/launcher/details/click-to-deploy-images/redis)) host reliable Redis services which you may consider for this purpose. A well supported Redis [Docker container](https://hub.docker.com/_/redis/) also exists if you would rather run Redis in a container. For more information on how to configure the `fleet` binary to use the correct Redis instance, see the [Configuration](./2-Configuration.md) document. +Fleet uses Redis to ingest and queue the results of distributed queries, cache data, etc. Many cloud providers (such as [AWS](https://aws.amazon.com/elasticache/) and [GCP](https://console.cloud.google.com/launcher/details/click-to-deploy-images/redis)) host reliable Redis services which you may consider for this purpose. A well supported Redis [Docker container](https://hub.docker.com/_/redis/) also exists if you would rather run Redis in a container. For more information on how to configure the `fleet` binary to use the correct Redis instance, see the [Configuration](./02-Configuration.md) document. diff --git a/docs/02-Deploying/03-Example-deployment-scenarios.md b/docs/02-Deploying/03-Example-deployment-scenarios.md index b1b004859e..7dde2f46d7 100644 --- a/docs/02-Deploying/03-Example-deployment-scenarios.md +++ b/docs/02-Deploying/03-Example-deployment-scenarios.md @@ -48,7 +48,7 @@ vagrant ssh ### Installing Fleet -To [install Fleet](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/1-Installation.md), download, unzip, and move the latest Fleet binary to your desired install location. +To [install Fleet](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/01-Installation.md), download, unzip, and move the latest Fleet binary to your desired install location. For example, after downloading: ```sh @@ -190,11 +190,11 @@ Now, if you go to [https://localhost:8080](https://localhost:8080) in your local ### Running Fleet with systemd -See [Running with systemd](./2-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. +See [Running with systemd](./02-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. ### Installing and running osquery -> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../1-Using-Fleet/4-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. +> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../01-Using-Fleet/04-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. To install osquery on CentOS, you can run the following: @@ -357,11 +357,11 @@ Now, if you go to [https://localhost:8080](https://localhost:8080) in your local ### Running Fleet with systemd -See [Running with systemd](./2-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. +See [Running with systemd](./02-Configuration.md#running-with-systemd) for documentation on running fleet as a background process and managing the fleet server logs. ### Installing and running osquery -> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../1-Using-Fleet/4-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. +> Note that this whole process is outlined in more detail in the [Adding Hosts To Fleet](../01-Using-Fleet/04-Adding-hosts.md) document. The steps are repeated here for the sake of a continuous tutorial. To install osquery on Ubuntu, you can run the following: @@ -453,14 +453,14 @@ We will use this address when we configure the Kubernetes deployment and databas The last step is to run the Fleet database migrations on your new MySQL server. To do this, run the following: ``` -kubectl create -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl create -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml ``` In Kubernetes, you can only run a job once. If you'd like to run it again (i.e.: you'd like to run the migrations again using the same file), you must delete the job before re-creating it. To delete the job and re-run it, you can run the following commands: ``` -kubectl delete -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml -kubectl create -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl delete -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml +kubectl create -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-migrations.yml ``` #### Redis @@ -536,7 +536,7 @@ ts=2017-11-16T02:48:38.441148166Z transport=https address=0.0.0.0:443 msg=listen Now that the Fleet server is running on our cluster, we have to expose the Fleet webservers to the internet via a load balancer. To create a Kubernetes `Service` of type `LoadBalancer`, run the following: ``` -kubectl apply -f ./docs/1-Using-Fleet/configuration-files/kubernetes/fleet-service.yml +kubectl apply -f ./docs/01-Using-Fleet/configuration-files/kubernetes/fleet-service.yml ``` #### Configure DNS diff --git a/docs/02-Deploying/FAQ.md b/docs/02-Deploying/FAQ.md index 4514a88d9b..f9580cbb68 100644 --- a/docs/02-Deploying/FAQ.md +++ b/docs/02-Deploying/FAQ.md @@ -25,7 +25,7 @@ Yes. Fleet scales horizontally out of the box as long as all of the Fleet server Note that osquery logs will be distributed across the Fleet servers. -Read the [performance documentation](../1-Using-Fleet/6-Monitoring-Fleet.md#fleet-server-performance) for more. +Read the [performance documentation](../01-Using-Fleet/06-Monitoring-Fleet.md#fleet-server-performance) for more. ## Why aren't my osquery agents connecting to Fleet? @@ -71,15 +71,15 @@ These configurations cannot be managed centrally from Fleet. ## What do I do about "too many open files" errors? -This error usually indicates that the Fleet server has run out of file descriptors. Fix this by increasing the `ulimit` on the Fleet process. See the `LimitNOFILE` setting in the [example systemd unit file](./2-Configuration.md#runing-with-systemd) for an example of how to do this with systemd. +This error usually indicates that the Fleet server has run out of file descriptors. Fix this by increasing the `ulimit` on the Fleet process. See the `LimitNOFILE` setting in the [example systemd unit file](./02-Configuration.md#runing-with-systemd) for an example of how to do this with systemd. -Some deployments may benefit by setting the [`--server_keepalive`](./2-Configuration.md#server_keepalive) flag to false. +Some deployments may benefit by setting the [`--server_keepalive`](./02-Configuration.md#server_keepalive) flag to false. This was also seen as a symptom of a different issue: if you're deploying on AWS on T type instances, there are different scenarios where the activity can increase and the instances will burst. If they run out of credits, then they'll stop processing leaving the file descriptors open. ## I upgraded my database, but Fleet is still running slowly. What could be going on? -This could be caused by a mismatched connection limit between the Fleet server and the MySQL server that prevents Fleet from fully utilizing the database. First [determine how many open connections your MySQL server supports](https://dev.mysql.com/doc/refman/8.0/en/too-many-connections.html). Now set the [`--mysql_max_open_conns`](./2-Configuration.md#mysql_max_open_conns) and [`--mysql_max_idle_conns`](./2-Configuration.md#mysql_max_idle_conns) flags appropriately. +This could be caused by a mismatched connection limit between the Fleet server and the MySQL server that prevents Fleet from fully utilizing the database. First [determine how many open connections your MySQL server supports](https://dev.mysql.com/doc/refman/8.0/en/too-many-connections.html). Now set the [`--mysql_max_open_conns`](./02-Configuration.md#mysql_max_open_conns) and [`--mysql_max_idle_conns`](./02-Configuration.md#mysql_max_idle_conns) flags appropriately. ## Why am I receiving a database connection error when attempting to "prepare" the database? diff --git a/docs/02-Deploying/README.md b/docs/02-Deploying/README.md index 19ad6389e1..5b62992f4b 100644 --- a/docs/02-Deploying/README.md +++ b/docs/02-Deploying/README.md @@ -1,15 +1,15 @@ # Deployment -### [Installation](./1-Installation.md) +### [Installation](./01-Installation.md) Provides documentation on installing the Fleet binary and Fleet’s infrastructure dependencies -### [Configuration](./2-Configuration.md) +### [Configuration](./02-Configuration.md) Includes resources for configuring the Fleet binary, managing osquery configurations, and running with systemd -### [Example deployment scenarios](./3-Example-deployment-scenarios.md) +### [Example deployment scenarios](./03-Example-deployment-scenarios.md) Includes deployment walkthroughs for Fleet on CentOS, Ubuntu, and Kubernetes. -### [Self-managed agent updates](./4-fleetctl-agent-updates.md) +### [Self-managed agent updates](./04-fleetctl-agent-updates.md) Information about running an update server with fleetctl. ### [FAQ](./FAQ.md) diff --git a/docs/03-Contributing/02-Testing.md b/docs/03-Contributing/02-Testing.md index 21e880f35d..1aae17579d 100644 --- a/docs/03-Contributing/02-Testing.md +++ b/docs/03-Contributing/02-Testing.md @@ -7,7 +7,7 @@ - [Test hosts](#test-hosts) - [Email](#email) - [Database backup/restore](#database-backuprestore) -- [Seeding Data](./6-Seeding-Data.md) +- [Seeding Data](./06-Seeding-Data.md) - [MySQL shell](#mysql-shell) - [Testing SSO](#testing-sso) @@ -137,7 +137,7 @@ E2E tests are run using Docker and Cypress. #### Preparation -Make sure dependencies are up to date and the [Fleet binaries are built locally](./1-Building-Fleet.md). +Make sure dependencies are up to date and the [Fleet binaries are built locally](./01-Building-Fleet.md). For Fleet Free tests: diff --git a/docs/03-Contributing/04-Committing-Changes.md b/docs/03-Contributing/04-Committing-Changes.md index 88694c9b38..73ed37ff02 100644 --- a/docs/03-Contributing/04-Committing-Changes.md +++ b/docs/03-Contributing/04-Committing-Changes.md @@ -19,7 +19,7 @@ Fleet Device Management team members may not copy queries from external sources Each developer (internal or external) creates a fork of the Fleet repository, committing changes to a branch within their fork. Changes are submitted by PR to be merged into Fleet. -GitHub Actions automatically runs testers and linters on each PR. Please ensure that these checks pass. Checks can be run locally as described in [2-Testing.md](./2-Testing.md). +GitHub Actions automatically runs testers and linters on each PR. Please ensure that these checks pass. Checks can be run locally as described in [02-Testing.md](./02-Testing.md). For features that are still in-progress, the Pull Request can be marked as a "Draft". This helps make it clear which PRs are ready for review and merge. diff --git a/docs/03-Contributing/FAQ.md b/docs/03-Contributing/FAQ.md index 4020cd0854..68c8d53b1b 100644 --- a/docs/03-Contributing/FAQ.md +++ b/docs/03-Contributing/FAQ.md @@ -25,7 +25,7 @@ server/fleet/emails.go:90:23: undefined: Asset make: *** [fleet] Error 2 ``` -If you get an `undefined: Asset` error it is likely because you did not run `make generate` before `make build`. See [Building Fleet](./1-Building-Fleet.md) for additional documentation on compiling the `fleet` binary. +If you get an `undefined: Asset` error it is likely because you did not run `make generate` before `make build`. See [Building Fleet](./01-Building-Fleet.md) for additional documentation on compiling the `fleet` binary. ## Adding hosts for testing diff --git a/docs/03-Contributing/README.md b/docs/03-Contributing/README.md index 159d3045b4..98a9b38856 100644 --- a/docs/03-Contributing/README.md +++ b/docs/03-Contributing/README.md @@ -1,18 +1,18 @@ # Contribution -### [Building Fleet](./1-Building-Fleet.md) +### [Building Fleet](./01-Building-Fleet.md) Provides documentation about building the code, development infrastructure, and database migrations -### [Testing](./2-Testing.md) +### [Testing](./02-Testing.md) Includes documentation about Fleet's full test suite and integration tests -### [Migrations](./3-Migrations.md) +### [Migrations](./03-Migrations.md) Information about creating and updating database migrations -### [Committing Changes](./4-Committing-Changes.md) +### [Committing Changes](./04-Committing-Changes.md) Contains information about how to merge changes into the codebase -### [Releasing Fleet](./5-Releasing-Fleet.md) +### [Releasing Fleet](./05-Releasing-Fleet.md) Provides a guide for Fleet's release process ### [FAQ](./FAQ.md) diff --git a/docs/README.md b/docs/README.md index 3da7649ef8..5e29c0c967 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,13 +2,13 @@ Welcome to the documentation for the Fleet osquery fleet manager. -### [Using Fleet](./1-Using-Fleet/README.md) +### [Using Fleet](./01-Using-Fleet/README.md) Resources for using the Fleet UI, fleetctl CLI, and Fleet REST API. -### [Deploying](./2-Deploying/README.md) +### [Deploying](./02-Deploying/README.md) Resources for installing Fleet's infrastructure dependencies, configuring Fleet, deploying osquery to hosts, and viewing example deployment scenarios. -### [Contributing](./3-Contributing/README.md) +### [Contributing](./03-Contributing/README.md) If you're interested in interacting with the Fleet source code, you'll find information on modifying and building the code here. --- diff --git a/frontend/README.md b/frontend/README.md index e7770abd47..147b24000f 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -17,7 +17,7 @@ The Fleet front-end is a Single Page Application using React with Typescript and ## Running the Fleet web app For details instruction on building and serving the Fleet web application -consult the [Contributing documentation](../docs/3-Contributing/README.md). +consult the [Contributing documentation](../docs/03-Contributing/README.md). ## Directory Structure diff --git a/frontend/README_deprecated.md b/frontend/README_deprecated.md index 4b1360d1da..79ac72db76 100644 --- a/frontend/README_deprecated.md +++ b/frontend/README_deprecated.md @@ -9,7 +9,7 @@ The Fleet front-end is a Single Page Application using React and Redux. ## Running the Fleet web app For details instruction on building and serving the Fleet web application -consult the [Contributing documentation](../docs/3-Contributing/README.md). +consult the [Contributing documentation](../docs/03-Contributing/README.md). ## Directory Structure diff --git a/handbook/README.md b/handbook/README.md index 3ea51a385e..d33d6ed6f6 100644 --- a/handbook/README.md +++ b/handbook/README.md @@ -98,7 +98,7 @@ If the action fails, please complete the following steps: ##### Browser compatibility checking -A browser compatibility check of [fleetdm.com](https://fleetdm.com/) should be carried out monthly to verify that the website looks, and functions as expected across all [supported browsers](../docs/1-Using-Fleet/12-Supported-browsers.md). +A browser compatibility check of [fleetdm.com](https://fleetdm.com/) should be carried out monthly to verify that the website looks, and functions as expected across all [supported browsers](../docs/01-Using-Fleet/12-Supported-browsers.md). - We use [BrowserStack](https://www.browserstack.com/users/sign_in) (logins can be found in [1Password](https://start.1password.com/open/i?a=N3F7LHAKQ5G3JPFPX234EC4ZDQ&v=3ycqkai6naxhqsylmsos6vairu&i=nwnxrrbpcwkuzaazh3rywzoh6e&h=fleetdevicemanagement.1password.com)) for our cross-browser checks. - Check for issues against the latest version of Google Chrome (macOS). We use this as our baseline for quality assurance. diff --git a/handbook/manual-qa.md b/handbook/manual-qa.md index 6af722b196..0c769e22e2 100644 --- a/handbook/manual-qa.md +++ b/handbook/manual-qa.md @@ -94,13 +94,13 @@ Add a query as a saved query to the pack. Remove this query. Delete the pack. As an admin user, select the "Settings" tab in the top navigation and then select "Organization settings". -Follow [the instructions outlined in the Testing documentation](../docs/3-Contributing/2-Testing.md#email) to set up a local SMTP server. +Follow [the instructions outlined in the Testing documentation](../docs/03-Contributing/02-Testing.md#email) to set up a local SMTP server. Successfully edit your organization's name in Fleet. ### Manage users flow -Invite a new user. To be able to invite users, you must have your local SMTP server configured. Instructions for setting up a local SMTP server are outlined in [the Testing documentation](../docs/3-Contributing/2-Testing.md#email) +Invite a new user. To be able to invite users, you must have your local SMTP server configured. Instructions for setting up a local SMTP server are outlined in [the Testing documentation](../docs/03-Contributing/02-Testing.md#email) Logout of your current admin user and accept the invitation for the newly invited user. With your local SMTP server configured, head to https://localhost:8025 to view and select the invitation link. diff --git a/handbook/release-process.md b/handbook/release-process.md index 52b5c6aaec..9e7d01f262 100644 --- a/handbook/release-process.md +++ b/handbook/release-process.md @@ -28,7 +28,7 @@ Check out the [Fleet 4.1.0 blog post](https://blog.fleetdm.com/fleet-4-1-0-57dfa **More improvements** - Includes each additional feature's name, availability (Free v. Premium), and 1-2 sentences that answer the 'why should the user care?' questions. -**Upgrade plan** - Once sentence that links to user to the upgrading Fleet documentation here: https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/8-Updating-Fleet.md +**Upgrade plan** - Once sentence that links to user to the upgrading Fleet documentation here: https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/08-Updating-Fleet.md ### Manual QA @@ -38,4 +38,4 @@ Documentation on conducting the manual QA pass can be found [here](./manual-qa.m ## Release day -Documentation on completing the release process can be found [here](../docs/3-Contributing/5-Releasing-Fleet.md). +Documentation on completing the release process can be found [here](../docs/03-Contributing/05-Releasing-Fleet.md). diff --git a/handbook/support-process.md b/handbook/support-process.md index cbacf8587c..58d09e964e 100644 --- a/handbook/support-process.md +++ b/handbook/support-process.md @@ -90,6 +90,6 @@ There are four sources that the individual on-call should monitor for activity: There are several locations in Fleet's public and internal documentation that can be helpful when answering questions raised by the community: -1. The frequently asked question (FAQ) documents in each section found in the `/docs` folder. These documents are the [Using Fleet FAQ](../docs/1-Using-Fleet/FAQ.md), [Deploying FAQ](../docs/2-Deploying/FAQ.md), and [Contributing FAQ](../docs/3-Contributing/FAQ.md). +1. The frequently asked question (FAQ) documents in each section found in the `/docs` folder. These documents are the [Using Fleet FAQ](../docs/01-Using-Fleet/FAQ.md), [Deploying FAQ](../docs/02-Deploying/FAQ.md), and [Contributing FAQ](../docs/03-Contributing/FAQ.md). 2. The [Internal FAQ](https://docs.google.com/document/d/1I6pJ3vz0EE-qE13VmpE2G3gd5zA1m3bb_u8Q2G3Gmp0/edit#heading=h.ltavvjy511qv) document. diff --git a/orbit/README.md b/orbit/README.md index 18cd740e52..3b3d1da3be 100644 --- a/orbit/README.md +++ b/orbit/README.md @@ -192,7 +192,7 @@ Yes! Orbit is licensed under an MIT license and all uses are encouraged. ### How does orbit update osquery? And how do the stable and edge channels get triggered to update osquery on a self hosted Fleet instance? -Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/main/docs/2-Deploying/4-fleetctl-agent-updates.md) as part of Fleet Premium (the subscription offering). +Orbit uses a configurable update server. We expect that many folks will just use the update server we manage (similar to what Kolide does with Launcher's update server). We are also offering [tooling for self-managing an update server](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/04-fleetctl-agent-updates.md) as part of Fleet Premium (the subscription offering). ## Community diff --git a/tools/backup_db/README.md b/tools/backup_db/README.md index 39f52ea042..11d7ad61a8 100644 --- a/tools/backup_db/README.md +++ b/tools/backup_db/README.md @@ -1,3 +1,3 @@ These scripts are for backing up and restore the Docker development MySQL database. -Usage is documented [here](../../docs/3-Contributing/2-Testing.md#database-backuprestore). +Usage is documented [here](../../docs/03-Contributing/02-Testing.md#database-backuprestore). diff --git a/tools/fleetctl-npm/README.md b/tools/fleetctl-npm/README.md index 52ff50a021..e303e10c50 100644 --- a/tools/fleetctl-npm/README.md +++ b/tools/fleetctl-npm/README.md @@ -10,4 +10,4 @@ Simply install `fleetctl` with `npm install -g fleetctl`. ## Usage -See the [fleetctl documentation](https://github.com/fleetdm/fleet/blob/main/docs/1-Using-Fleet/2-fleetctl-CLI.md) or `fleetctl --help` for usage instructions. +See the [fleetctl documentation](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/02-fleetctl-CLI.md) or `fleetctl --help` for usage instructions. From e03b2c7ee32e54155be14bcfbd9ad566abbe7187 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Wed, 22 Sep 2021 09:53:41 -0700 Subject: [PATCH 56/82] Remove erroneously included infra files (#2179) Nothing sensitive was included. --- tools/deploy/terraform-aws-fargate/database.tf | 12 ------------ tools/deploy/terraform-aws-fargate/variables.tf | 1 - tools/docker-fleetctl-awscli/Dockerfile | 4 ---- 3 files changed, 17 deletions(-) delete mode 100644 tools/deploy/terraform-aws-fargate/database.tf delete mode 100644 tools/deploy/terraform-aws-fargate/variables.tf delete mode 100644 tools/docker-fleetctl-awscli/Dockerfile diff --git a/tools/deploy/terraform-aws-fargate/database.tf b/tools/deploy/terraform-aws-fargate/database.tf deleted file mode 100644 index c87da3ade7..0000000000 --- a/tools/deploy/terraform-aws-fargate/database.tf +++ /dev/null @@ -1,12 +0,0 @@ -resource "aws_db_instance" "default" { - allocated_storage = 10 - engine = "mysql" - engine_version = "5.7" - instance_class = "db.t3.micro" - identifier_prefix = "fleet" - name = "fleet" - username = "foo" - password = "foobarbaz" - parameter_group_name = "default.mysql5.7" - skip_final_snapshot = true -} \ No newline at end of file diff --git a/tools/deploy/terraform-aws-fargate/variables.tf b/tools/deploy/terraform-aws-fargate/variables.tf deleted file mode 100644 index b7bf843b5d..0000000000 --- a/tools/deploy/terraform-aws-fargate/variables.tf +++ /dev/null @@ -1 +0,0 @@ -variable "vpc_id" {} \ No newline at end of file diff --git a/tools/docker-fleetctl-awscli/Dockerfile b/tools/docker-fleetctl-awscli/Dockerfile deleted file mode 100644 index ff1fed9368..0000000000 --- a/tools/docker-fleetctl-awscli/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM amazon/aws-cli -MAINTAINER Fleet Developers - -RUN curl https://github.com/fleetdm/fleet/releases/latest/download/fleetctl-linux.tar.gz | tar -xf From 3ea0439cf03736af8f5f7122d87fec8c2945c282 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 22 Sep 2021 14:44:45 -0400 Subject: [PATCH 57/82] Document the recommended max lifetime config for read replicas (#2189) --- docs/02-Deploying/02-Configuration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/02-Deploying/02-Configuration.md b/docs/02-Deploying/02-Configuration.md index 0b2f95938f..c959381768 100644 --- a/docs/02-Deploying/02-Configuration.md +++ b/docs/02-Deploying/02-Configuration.md @@ -131,7 +131,9 @@ All duration-based settings accept valid time units of `s`, `m`, `h`. ##### MySQL -This section describes the configuration options for the primary - if you also want to setup a read replica, the options are the same, except that the yaml section is `mysql_read_replica`, and the flags have the `mysql_read_replica_` prefix instead of `mysql_` (the corresponding environment variables follow the same transformation). Note that there is no default value for `mysql_read_replica_address`, it must be set explicitly for fleet to use a read replica. +This section describes the configuration options for the primary - if you also want to setup a read replica, the options are the same, except that the yaml section is `mysql_read_replica`, and the flags have the `mysql_read_replica_` prefix instead of `mysql_` (the corresponding environment variables follow the same transformation). Note that there is no default value for `mysql_read_replica_address`, it must be set explicitly for fleet to use a read replica, and it is recommended in that case to set a non-zero value for `mysql_read_replica_conn_max_lifetime` as in some environments, the replica's address may dynamically change to point +from the primary to an actual distinct replica based on auto-scaling options, so existing idle connections need to be recycled +periodically. ###### mysql_address From 8600d71d353b67992712cbce54fd9edffc4ca1a8 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 22 Sep 2021 17:18:55 -0300 Subject: [PATCH 58/82] Add osquery perf (#2190) * Add osquery perf * Update dockerfile and gh action --- .../workflows/push-osquery-perf-to-ecr.yml | 39 +++ Dockerfile.osquery-perf | 16 + cmd/osquery-perf/README.md | 82 +++++ cmd/osquery-perf/agent.go | 285 ++++++++++++++++ cmd/osquery-perf/mac10.14.6.tmpl | 309 ++++++++++++++++++ go.mod | 2 +- go.sum | 2 + 7 files changed, 734 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/push-osquery-perf-to-ecr.yml create mode 100644 Dockerfile.osquery-perf create mode 100644 cmd/osquery-perf/README.md create mode 100644 cmd/osquery-perf/agent.go create mode 100644 cmd/osquery-perf/mac10.14.6.tmpl diff --git a/.github/workflows/push-osquery-perf-to-ecr.yml b/.github/workflows/push-osquery-perf-to-ecr.yml new file mode 100644 index 0000000000..9ac308d0bb --- /dev/null +++ b/.github/workflows/push-osquery-perf-to-ecr.yml @@ -0,0 +1,39 @@ +name: Build docker image and publish to ECR + +on: + workflow_dispatch: + inputs: + enroll_secret: + description: 'Enroll Secret' + required: true + url: + description: 'Fleet server URL' + required: true + tag: + description: 'docker image tag' + required: true + default: latest + +jobs: + build-docker: + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.LOADTEST_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.LOADTEST_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: osquery-perf + IMAGE_TAG: ${{ github.event.inputs.tag }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --build-arg ENROLL_SECRET=${{ github.event.inputs.enroll_secret }} --build-arg HOST_COUNT=${{ github.event.inputs.host_count }} --build-arg SERVER_URL=${{ github.event.inputs.url }} -f Dockerfile.osquery-perf . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file diff --git a/Dockerfile.osquery-perf b/Dockerfile.osquery-perf new file mode 100644 index 0000000000..e78d967ef5 --- /dev/null +++ b/Dockerfile.osquery-perf @@ -0,0 +1,16 @@ +FROM golang:1.17.1-alpine + +ARG ENROLL_SECRET +ARG HOST_COUNT +ARG SERVER_URL + +ENV ENROLL_SECRET ${ENROLL_SECRET} +ENV HOST_COUNT ${HOST_COUNT} +ENV SERVER_URL ${SERVER_URL} + +COPY ./cmd/osquery-perf/agent.go ./go.mod ./go.sum ./cmd/osquery-perf/mac10.14.6.tmpl /osquery-perf/ +WORKDIR /osquery-perf/ +RUN go mod download +RUN go build -o osquery-perf + +CMD ./osquery-perf -enroll_secret $ENROLL_SECRET -host_count $HOST_COUNT -server_url $SERVER_URL \ No newline at end of file diff --git a/cmd/osquery-perf/README.md b/cmd/osquery-perf/README.md new file mode 100644 index 0000000000..9a809d34ae --- /dev/null +++ b/cmd/osquery-perf/README.md @@ -0,0 +1,82 @@ +# Osquery Server Performance Tester + +> **TODO: Archive this repo and move its contents inline into https://github.com/fleetdm/fleet** + +This repository provides a tool to generate realistic traffic to an osquery +management server (primarily, [Fleet](https://github.com/fleetdm/fleet)). With +this tool, many thousands of hosts can be simulated from a single host. + +## Requirements + +The only requirement for running this tool is a working installation of +[Go](https://golang.org/doc/install). + +## Usage + +Typically `go run` is used: + +``` +go run agent.go --help +Usage of agent.go: + -config_interval duration + Interval for config requests (default 1m0s) + -enroll_secret string + Enroll secret to authenticate enrollment + -host_count int + Number of hosts to start (default 10) (default 10) + -query_interval duration + Interval for live query requests (default 10s) + -seed int + Seed for random generator (default current time) (default 1586310930917739000) + -server_url string + URL (with protocol and port of osquery server) (default "https://localhost:8080") + -start_period duration + Duration to spread start of hosts over (default 10s) +exit status 2 +``` + +The tool should be invoked with the appropriate enroll secret. A typical +invocation looks like: + +``` +go run agent.go --enroll_secret hgh4hk3434l2jjf +``` + +When starting many hosts, it is a good idea to extend the intervals, and also +the period over which the hosts are started: + +``` +go run agent.go --enroll_secret hgh4hk3434l2jjf --host_count 5000 --start_period 5m --query_interval 60s --config_interval 5m +``` + +This will start 5,000 hosts over a period of 5 minutes. Each host will check in +for live queries at a 1 minute interval, and for configuration at a 5 minute +interval. Starting over a 5 minute period ensures that the configuration +requests are spread evenly over the 5 minute interval. + +It can be useful to start the "same" hosts. This can be achieved with the +`--seed` parameter: + +``` +go run agent.go --enroll_secret hgh4hk3434l2jjf --seed 0 +``` + +By using the same seed, along with other values, we usually get hosts that look +the same to the server. This is not guaranteed, but it is a useful technique. + +### Resource Limits + +On many systems, trying to simulate a large number of hosts will result in hitting system resource limits (such as number of open file descriptors). + +If you see errors such as `dial tcp: lookup localhost: no such host` or `read: connection reset by peer`, try increasing these limits. + +#### macOS + +Run the following command in the shell before running the Fleet server _and_ before running `agent.go` (run it once in each shell): + +``` sh +ulimit -n 64000 +``` + +## Bugs +To report a bug, [click here](https://github.com/fleetdm/fleet). diff --git a/cmd/osquery-perf/agent.go b/cmd/osquery-perf/agent.go new file mode 100644 index 0000000000..a0339bd8fa --- /dev/null +++ b/cmd/osquery-perf/agent.go @@ -0,0 +1,285 @@ +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "math/rand" + "net/http" + "strings" + "text/template" + "time" + + "github.com/google/uuid" +) + +type Agent struct { + ServerAddress string + EnrollSecret string + NodeKey string + UUID string + Client http.Client + ConfigInterval time.Duration + QueryInterval time.Duration + Templates *template.Template + strings map[string]string +} + +func NewAgent(serverAddress, enrollSecret string, templates *template.Template, configInterval, queryInterval time.Duration) *Agent { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + transport.DisableCompression = true + return &Agent{ + ServerAddress: serverAddress, + EnrollSecret: enrollSecret, + Templates: templates, + ConfigInterval: configInterval, + QueryInterval: queryInterval, + UUID: uuid.New().String(), + Client: http.Client{Transport: transport}, + strings: make(map[string]string), + } +} + +type enrollResponse struct { + NodeKey string `json:"node_key"` +} + +type distributedReadResponse struct { + Queries map[string]string `json:"queries"` +} + +func (a *Agent) runLoop() { + a.Enroll() + + a.Config() + resp, err := a.DistributedRead() + if err != nil { + log.Println(err) + } else { + if len(resp.Queries) > 0 { + a.DistributedWrite(resp.Queries) + } + } + + configTicker := time.Tick(a.ConfigInterval) + liveQueryTicker := time.Tick(a.QueryInterval) + for { + select { + case <-configTicker: + a.Config() + case <-liveQueryTicker: + resp, err := a.DistributedRead() + if err != nil { + log.Println(err) + } else { + if len(resp.Queries) > 0 { + a.DistributedWrite(resp.Queries) + } + } + } + } +} + +const stringVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_." + +func (a *Agent) randomString(n int) string { + sb := strings.Builder{} + sb.Grow(n) + for i := 0; i < n; i++ { + sb.WriteByte(stringVals[rand.Int63()%int64(len(stringVals))]) + } + return sb.String() +} + +func (a *Agent) CachedString(key string) string { + if val, ok := a.strings[key]; ok { + return val + } + val := a.randomString(12) + a.strings[key] = val + return val +} + +func (a *Agent) Enroll() { + var body bytes.Buffer + if err := a.Templates.ExecuteTemplate(&body, "enroll", a); err != nil { + log.Println("execute template:", err) + return + } + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/enroll", &body) + if err != nil { + log.Println("create request:", err) + return + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + log.Println("do request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Println("status:", resp.Status) + return + } + + var parsedResp enrollResponse + if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil { + log.Println("json parse:", err) + return + } + + a.NodeKey = parsedResp.NodeKey +} + +func (a *Agent) Config() { + body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`) + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/config", body) + if err != nil { + log.Println("create config request:", err) + return + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + log.Println("do config request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Println("config status:", resp.Status) + return + } + + // No need to read the config body +} + +func (a *Agent) DistributedRead() (*distributedReadResponse, error) { + body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`) + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/read", body) + if err != nil { + return nil, fmt.Errorf("create distributed read request: %s", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("do distributed read request: %s", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("distributed read status: %s", resp.Status) + } + + var parsedResp distributedReadResponse + if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil { + return nil, fmt.Errorf("json parse distributed read response: %s", err) + } + + return &parsedResp, nil +} + +type distributedWriteRequest struct { + Queries map[string]json.RawMessage `json:"queries"` + Statuses map[string]string `json:"statuses"` + NodeKey string `json:"node_key"` +} + +var defaultQueryResult = json.RawMessage(`[{"foo": "bar"}]`) + +const statusSuccess = "0" + +func (a *Agent) DistributedWrite(queries map[string]string) { + var body bytes.Buffer + + if _, ok := queries["fleet_detail_query_network_interface"]; ok { + // Respond to label/detail queries + a.Templates.ExecuteTemplate(&body, "distributed_write", a) + } else { + // Return a generic response for any other queries + req := distributedWriteRequest{ + Queries: make(map[string]json.RawMessage), + Statuses: make(map[string]string), + NodeKey: a.NodeKey, + } + + for name := range queries { + req.Queries[name] = defaultQueryResult + req.Statuses[name] = statusSuccess + } + json.NewEncoder(&body).Encode(req) + } + + req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/write", &body) + if err != nil { + log.Println("create distributed write request:", err) + return + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "osquery/4.6.0") + + resp, err := a.Client.Do(req) + if err != nil { + log.Println("do distributed write request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Println("distributed write status:", resp.Status) + return + } + + // No need to read the distributed write body +} + +func main() { + serverURL := flag.String("server_url", "https://localhost:8080", "URL (with protocol and port of osquery server)") + enrollSecret := flag.String("enroll_secret", "", "Enroll secret to authenticate enrollment") + hostCount := flag.Int("host_count", 10, "Number of hosts to start (default 10)") + randSeed := flag.Int64("seed", time.Now().UnixNano(), "Seed for random generator (default current time)") + startPeriod := flag.Duration("start_period", 10*time.Second, "Duration to spread start of hosts over") + configInterval := flag.Duration("config_interval", 1*time.Minute, "Interval for config requests") + queryInterval := flag.Duration("query_interval", 10*time.Second, "Interval for live query requests") + + flag.Parse() + + rand.Seed(*randSeed) + + tmpl, err := template.ParseGlob("*.tmpl") + if err != nil { + log.Fatal("parse templates: ", err) + } + + // Spread starts over the interval to prevent thunering herd + sleepTime := *startPeriod / time.Duration(*hostCount) + var agents []*Agent + for i := 0; i < *hostCount; i++ { + a := NewAgent(*serverURL, *enrollSecret, tmpl, *configInterval, *queryInterval) + agents = append(agents, a) + go a.runLoop() + time.Sleep(sleepTime) + } + + fmt.Println("Agents running. Kill with C-c.") + <-make(chan struct{}) +} diff --git a/cmd/osquery-perf/mac10.14.6.tmpl b/cmd/osquery-perf/mac10.14.6.tmpl new file mode 100644 index 0000000000..88b6f713a9 --- /dev/null +++ b/cmd/osquery-perf/mac10.14.6.tmpl @@ -0,0 +1,309 @@ +{{ define "enroll" -}} +{ + "enroll_secret": "{{ .EnrollSecret }}", + "host_details": { + "os_version": { + "build": "18G3020", + "major": "10", + "minor": "14", + "name": "Mac OS X", + "patch": "6", + "platform": "darwin", + "platform_like": "darwin", + "version": "10.14.6" + }, + "osquery_info": { + "build_distro": "10.12", + "build_platform": "darwin", + "config_hash": "", + "config_valid": "0", + "extensions": "inactive", + "instance_id": "{{ .UUID }}", + "pid": "12947", + "platform_mask": "21", + "start_time": "1580931224", + "uuid": "{{ .UUID }}", + "version": "4.6.0", + "watcher": "12946" + }, + "platform_info": { + "address": "0xff990000", + "date": "12/16/2019 ", + "extra": "MBP114; 196.0.0.0.0; root@xapp160; Mon Dec 16 15:55:18 PST 2019; 196 (B&I); F000_B00; Official Build, Release; Apple LLVM version 5.0 (clang-500.0.68) (based on LLVM 3.3svn)", + "revision": "196 (B&I)", + "size": "8388608", + "vendor": "Apple Inc. ", + "version": "196.0.0.0.0 ", + "volume_size": "1507328" + }, + "system_info": { + "computer_name": "{{ .CachedString "hostname" }}", + "cpu_brand": "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "cpu_logical_cores": "8", + "cpu_physical_cores": "4", + "cpu_subtype": "Intel x86-64h Haswell", + "cpu_type": "x86_64h", + "hardware_model": "MacBookPro11,4", + "hardware_serial": "D02R835DG8WK", + "hardware_vendor": "Apple Inc.", + "hardware_version": "1.0", + "hostname": "{{ .CachedString "hostname" }}", + "local_hostname": "{{ .CachedString "hostname" }}", + "physical_memory": "17179869184", + "uuid": "{{ .UUID }}" + } + }, + "host_identifier": "{{ .CachedString "hostname" }}", + "platform_type": "21" +} +{{- end }} + +{{ define "distributed_write" -}} +{ + "queries":{ + "fleet_detail_query_network_interface":[ + { + "point_to_point":"", + "address":"fe80::8cb:112d:ff51:1e5d%en0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"en0", + "mac":"f8:2d:88:93:56:5c", + "type":"6", + "mtu":"1500", + "metric":"0", + "ipackets":"278493", + "opackets":"206238", + "ibytes":"275799040", + "obytes":"37720064", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582848084" + +}, + { + "point_to_point":"", + "address":"192.168.1.3", + "mask":"255.255.255.0", + "broadcast":"192.168.1.255", + "interface":"en0", + "mac":"f5:5a:80:92:52:5b", + "type":"6", + "mtu":"1500", + "metric":"0", + "ipackets":"278493", + "opackets":"206238", + "ibytes":"275799040", + "obytes":"37720064", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582848084" + +}, + { + "point_to_point":"127.0.0.1", + "address":"127.0.0.1", + "mask":"255.0.0.0", + "broadcast":"", + "interface":"lo0", + "mac":"00:00:00:00:00:00", + "type":"24", + "mtu":"16384", + "metric":"0", + "ipackets":"132952", + "opackets":"132952", + "ibytes":"67053568", + "obytes":"67053568", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840871" + +}, + { + "point_to_point":"::1", + "address":"::1", + "mask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "broadcast":"", + "interface":"lo0", + "mac":"00:00:00:00:00:00", + "type":"24", + "mtu":"16384", + "metric":"0", + "ipackets":"132952", + "opackets":"132952", + "ibytes":"67053568", + "obytes":"67053568", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840871" + +}, + { + "point_to_point":"", + "address":"fe80::1%lo0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"lo0", + "mac":"00:00:00:00:00:00", + "type":"24", + "mtu":"16384", + "metric":"0", + "ipackets":"132952", + "opackets":"132952", + "ibytes":"67053568", + "obytes":"67053568", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840871" + +}, + { + "point_to_point":"", + "address":"fe80::3a:84ff:fe6b:bf75%awdl0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"awdl0", + "mac":"03:3b:94:5b:be:75", + "type":"6", + "mtu":"1484", + "metric":"0", + "ipackets":"0", + "opackets":"16", + "ibytes":"0", + "obytes":"3072", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582842892" + +}, + { + "point_to_point":"", + "address":"fe80::6eaf:9721:3476:b691%utun0", + "mask":"ffff:ffff:ffff:ffff::", + "broadcast":"", + "interface":"utun0", + "mac":"00:00:00:00:00:00", + "type":"1", + "mtu":"2000", + "metric":"0", + "ipackets":"0", + "opackets":"2", + "ibytes":"0", + "obytes":"0", + "ierrors":"0", + "oerrors":"0", + "idrops":"0", + "odrops":"0", + "last_change":"1582840897" + +} + +], + "fleet_detail_query_os_version":[ + { + "name":"Mac OS X", + "version":"10.14.6", + "major":"10", + "minor":"14", + "patch":"6", + "build":"18G3020", + "platform":"darwin", + "platform_like":"darwin", + "codename":"" + +} + +], + "fleet_detail_query_osquery_flags":[ + { + "name":"config_refresh", + "value":"{{ printf "%.0f" .ConfigInterval.Seconds }}" + +}, + { + "name":"distributed_interval", + "value":"{{ printf "%.0f" .QueryInterval.Seconds }}" + +}, + { + "name":"logger_tls_period", + "value":"99999" + +} + +], + "fleet_detail_query_osquery_info":[ + { + "pid":"11287", + "uuid":"{{ .UUID }}", + "instance_id":"{{ .UUID }}", + "version":"4.1.2", + "config_hash":"b01efbf375ac6767f259ae98751154fef727ce35", + "config_valid":"1", + "extensions":"inactive", + "build_platform":"darwin", + "build_distro":"10.12", + "start_time":"1582857555", + "watcher":"11286", + "platform_mask":"21" + +} + +], + "fleet_detail_query_system_info":[ + { + "hostname":"{{ .CachedString "hostname" }}", + "uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB", + "cpu_type":"x86_64h", + "cpu_subtype":"Intel x86-64h Haswell", + "cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", + "cpu_physical_cores":"4", + "cpu_logical_cores":"8", + "cpu_microcode":"", + "physical_memory":"17179869184", + "hardware_vendor":"Apple Inc.", + "hardware_model":"MacBookPro11,4", + "hardware_version":"1.0", + "hardware_serial":"C02R262BM8LN", + "computer_name":"{{ .CachedString "hostname" }}", + "local_hostname":"{{ .CachedString "hostname" }}" + +} + +], + "fleet_detail_query_uptime":[ + { + "days":"0", + "hours":"4", + "minutes":"38", + "seconds":"11", + "total_seconds":"16691" + +} + +] + +}, + "statuses":{ + "fleet_detail_query_network_interface":0, + "fleet_detail_query_os_version":0, + "fleet_detail_query_osquery_flags":0, + "fleet_detail_query_osquery_info":0, + "fleet_detail_query_system_info":0, + "fleet_detail_query_uptime":0 +}, + "node_key":"{{ .NodeKey }}" +} +{{- end }} diff --git a/go.mod b/go.mod index b7b7271608..6b0d2d7ea9 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/gomodule/redigo v1.8.5 github.com/google/go-cmp v0.5.6 github.com/google/go-github/v37 v37.0.0 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.3.0 github.com/goreleaser/nfpm/v2 v2.2.2 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum index dd625a9fa6..c045b59475 100644 --- a/go.sum +++ b/go.sum @@ -454,6 +454,8 @@ github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4Mgqvf github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= From 1db2acbff7f5ca8792501438822918161521c114 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 22 Sep 2021 17:21:50 -0300 Subject: [PATCH 59/82] Add host count input (#2191) --- .github/workflows/push-osquery-perf-to-ecr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/push-osquery-perf-to-ecr.yml b/.github/workflows/push-osquery-perf-to-ecr.yml index 9ac308d0bb..2841658a4e 100644 --- a/.github/workflows/push-osquery-perf-to-ecr.yml +++ b/.github/workflows/push-osquery-perf-to-ecr.yml @@ -9,6 +9,10 @@ on: url: description: 'Fleet server URL' required: true + host_count: + description: 'Amount of hosts to emulate' + required: true + default: 20 tag: description: 'docker image tag' required: true From 07342e95c95fc69911d79687275ef76347334860 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 22 Sep 2021 17:26:02 -0300 Subject: [PATCH 60/82] Need to checkout code explicitly (#2192) --- .github/workflows/push-osquery-perf-to-ecr.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push-osquery-perf-to-ecr.yml b/.github/workflows/push-osquery-perf-to-ecr.yml index 2841658a4e..197ec32864 100644 --- a/.github/workflows/push-osquery-perf-to-ecr.yml +++ b/.github/workflows/push-osquery-perf-to-ecr.yml @@ -22,6 +22,9 @@ jobs: build-docker: runs-on: ubuntu-latest steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From 2505b68cb8dd82437a51a109d503477c35c2e484 Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Wed, 22 Sep 2021 23:15:25 -0500 Subject: [PATCH 61/82] Make logos page more inviting for anyone in the community who might want to use our logo in an article or apply a wallpaper (#2174) --- website/views/layouts/layout.ejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/views/layouts/layout.ejs b/website/views/layouts/layout.ejs index d609febe48..4c3854afb4 100644 --- a/website/views/layouts/layout.ejs +++ b/website/views/layouts/layout.ejs @@ -135,7 +135,7 @@ Blog Jobs Contribute - Press Kit + Logos/artwork Hall of fame
    From e5452f4e4de1095db7d3a378d807026a44514475 Mon Sep 17 00:00:00 2001 From: Renee Jackson <44620612+rlynnj11@users.noreply.github.com> Date: Thu, 23 Sep 2021 11:32:02 -0300 Subject: [PATCH 62/82] Update manual qa document (#2124) * Pack location Future-proofing instructions for ease of onboards by adding location information. * Label queries are immutable (addition) Previous document made it seem users should be able to edit label queries. * removal of all non-preview tasks Non-preview tasks will be moved to /docs/3-Contributing/2-Testing.md. Additional changes include clarifications to the testing steps. * update link * Update manual-qa.md --- handbook/manual-qa.md | 126 ++++-------------------------------------- 1 file changed, 10 insertions(+), 116 deletions(-) diff --git a/handbook/manual-qa.md b/handbook/manual-qa.md index 0c769e22e2..3f84eb2412 100644 --- a/handbook/manual-qa.md +++ b/handbook/manual-qa.md @@ -2,63 +2,31 @@ This living document outlines the manual quality assurance process conducted to ensure each release of Fleet meets organization standards. -All steps should be conducted during each QA pass. +All steps should be conducted during each QA pass. All steps are possible with `fleetctrl preview`. As new features are added to Fleet, new steps and flows will be added. ## Collecting bugs -The goal of manual QA is to catch unexpected behavior prior to release. +The goal of manual QA is to catch unexpected behavior prior to release. All Manual QA steps should be possible using `fleetctl preview`. Please refer to [docs/03-Contributing/02-Testing.md](https://github.com/fleetdm/fleet/blob/main/docs/03-Contributing/02-Testing.md) for flows that cannot be completed using `fleetctl preview`. Please start the manual QA process by creating a blank GitHub issue. As you complete each of the flows, record a list of the bugs you encounter in this new issue. Each item in this list should contain one sentence describing the bug and a screenshot if the item is a frontend bug. ## Fleet UI -### Clear your local MySQL database - -Before you fire up your local Fleet server, wipe your local MySQL database by running the following command: - -``` -docker volume rm fleet_mysql-persistent-volume -``` - -If you receive an error that says "No such volume," double check that the MySQL volume doesn't have a different name by running this command: - -``` -docker volume ls -``` - -### Start your development server - -Next, fire up your local Fleet server. Check out [this Loom video](https://www.loom.com/share/e7439f058eb44c45af872abe8f8de4a1) for instructions on starting up your local development environment. +For all following flows, please refer to the [permissions documentation](https://fleetdm.com/docs/using-fleet/permissions) to ensure that actions are limited to the appropriate user type. Any users with access beyond what this document lists as availale should be considered a bug and reported for either documentation updates or investigation. ### Set up flow -Successfully set up Fleet. +Successfully set up `fleetctl preview` using the preview steps outlined [here](https://fleetdm.com/get-started) ### Login and logout flow Successfully logout and then login to your local Fleet. -### Enroll host flow - -Enroll your local machine to Fleet. Check out the [Orbit for osquery documentation](https://github.com/fleetdm/orbit#orbit-osquery) for instructions on generating and installing an Orbit package. - -### Host page - -To populate the Fleet UI with more than just one host you'll need to use the [fleetdm/osquery-perf tool](https://github.com/fleetdm/osquery-perf/tree/629a7efb6097f9108f706ccd45828793ff73cf9c). - -First, clone the fleetdm/osquery perf repo and then run the following commands from the top level of the cloned directory: - -``` -go run agent.go --host_count 200 --enroll_secret -``` - -After about 10 seconds, the Fleet UI should be populated with 200 simulated hosts. - ### Host details page -Select a host from the "Hosts" table as a global user with the Maintainer role. +Select a host from the "Hosts" table as a global user with the Maintainer role. You may create a user with a fake email for this purpose. You should be able to see and select the "Delete" button on this host's **Host details** page. @@ -70,7 +38,9 @@ You should be able to see and select the "Query" button on this host's **Host de Create a new label by selecting "Add a new label" on the Hosts page. Make sure it correctly filters the host on the hosts page. -Edit this label and then delete this label. +Edit this label. Confirm users can only edit the "Name" and "Description" fields for a label. Users cannot edit the "Query" field because label queries are immutable. + +Delete this label. ### Query flow @@ -86,88 +56,12 @@ Edit this query and then delete this query. `Flow is covered by e2e testing` -Create a new pack. +Create a new pack (under Schedule/advanced). Add a query as a saved query to the pack. Remove this query. Delete the pack. -### Organization settings flow - -As an admin user, select the "Settings" tab in the top navigation and then select "Organization settings". - -Follow [the instructions outlined in the Testing documentation](../docs/03-Contributing/02-Testing.md#email) to set up a local SMTP server. - -Successfully edit your organization's name in Fleet. - -### Manage users flow - -Invite a new user. To be able to invite users, you must have your local SMTP server configured. Instructions for setting up a local SMTP server are outlined in [the Testing documentation](../docs/03-Contributing/02-Testing.md#email) - -Logout of your current admin user and accept the invitation for the newly invited user. With your local SMTP server configured, head to https://localhost:8025 to view and select the invitation link. - -### Agent options flow - -Head to the global agent options page and set the `distributed_iterval` field to `5`. - -Refresh the page to confirm that the agent options have been updated. ### My account flow -Head to the My account page by selecting the dropdown icon next to your avatar in the top navigation. Select "My account" and successfully update your password. +Head to the My account page by selecting the dropdown icon next to your avatar in the top navigation. Select "My account" and successfully update your password. Please do this with an extra user created for this purpose to maintain accessibility of `fleetctl preview` admin user. -## `fleetctl` CLI - -### Set up flow - -Successfully set up Fleet by running the `fleetctl setup` command. - -You may have to wipe your local MySQL database in order to successfully set up Fleet. Check out the [Clear your local MySQL database](#clear-your-local-mysql-database) section of this document for instructions. - -### Login and logout flow - -Successfully login by running the `fleetctl login` command. - -Successfully logout by running the `fleetctl logout` command. Then, log in again. - -### Hosts - -Run the `fleetctl get hosts` command. - -You should see your local machine returned. If your host isn't showing up, you may have to reenroll your local machine. Check out the [Orbit for osquery documentation](https://github.com/fleetdm/fleet/blob/main/orbit/README.md) for instructions on generating and installing an Orbit package. - -### Query flow - -Apply the standard query library by running the following command: - -`fleetctl apply -f docs/1-Using-Fleet/standard-query-library/standard-query-library.yml` - -Make sure all queries were successfully added by running the following command: - -`fleetctl get queries` - -Run the "Get the version of the resident operating system" query against your local machine by running the following command: - -`fleetctl query --hosts --query-name "Get the version of the resident operating system"` - -### Pack flow - -Apply a pack by running the following commands: - -`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/queries.yml` - -`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/pack.yml` - -Make sure the pack was successfully added by running the following command: - -`fleetctl get packs` - -### Organization settings flow - -Apply organization settings by running the following command: - -`fleetctl apply -f docs/1-Using-Fleet/configuration-files/multi-file-configuration/organization-settings.yml` - -### Manage users flow - -Create a new user by running the `fleetctl user create` command. - -Logout of your current user and log in with the newly created user. From 8931163882efd72808ad8fb4a8ae82b9b7f6168b Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 23 Sep 2021 12:44:04 -0300 Subject: [PATCH 63/82] Don't check authViewer if there's no bearer token (#2200) --- changes/reduce-sql-queries-per-host | 1 + server/service/http_auth.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/reduce-sql-queries-per-host diff --git a/changes/reduce-sql-queries-per-host b/changes/reduce-sql-queries-per-host new file mode 100644 index 0000000000..833ebadfca --- /dev/null +++ b/changes/reduce-sql-queries-per-host @@ -0,0 +1 @@ +* Reduce the amount of queries to the database when a host checks in. diff --git a/server/service/http_auth.go b/server/service/http_auth.go index bf61a4c616..7a1c1756ef 100644 --- a/server/service/http_auth.go +++ b/server/service/http_auth.go @@ -18,9 +18,11 @@ func setRequestsContexts(svc fleet.Service) kithttp.RequestFunc { return func(ctx context.Context, r *http.Request) context.Context { bearer := token.FromHTTPRequest(r) ctx = token.NewContext(ctx, bearer) - v, err := authViewer(ctx, string(bearer), svc) - if err == nil { - ctx = viewer.NewContext(ctx, *v) + if bearer != "" { + v, err := authViewer(ctx, string(bearer), svc) + if err == nil { + ctx = viewer.NewContext(ctx, *v) + } } // get the user-id for request From a859d46af96a262fa16ee582e4457c701343ad49 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Thu, 23 Sep 2021 08:50:34 -0700 Subject: [PATCH 64/82] Fixes to fleetctl debug archive docs (#2203) --- docs/01-Using-Fleet/06-Monitoring-Fleet.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/01-Using-Fleet/06-Monitoring-Fleet.md b/docs/01-Using-Fleet/06-Monitoring-Fleet.md index 391db88122..f7534f6a01 100644 --- a/docs/01-Using-Fleet/06-Monitoring-Fleet.md +++ b/docs/01-Using-Fleet/06-Monitoring-Fleet.md @@ -68,7 +68,7 @@ For performance issues in the Fleet server process, please [file an issue](https ##### Generate debug archive (Fleet 3.4.0+) -Use the `fleetctl archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./02-fleetctl-CLI.md)) for details on configuring `fleetctl`. +Use the `fleetctl debug archive` command to generate an archive of Fleet's full suite of debug profiles. See the [fleetctl setup guide](./02-fleetctl-CLI.md)) for details on configuring `fleetctl`. The generated `.tar.gz` archive will be available in the current directory. @@ -86,4 +86,4 @@ fleetctl debug archive --context server-a ###### Confidential information -The `fleetctl archive` command retrieves information generated by Go's [`net/http/pprof`](https://golang.org/pkg/net/http/pprof/) package. In most scenarios this should not include sensitive information, however it does include command line arguments to the Fleet server. If the Fleet server receives sensitive credentials via CLI argument (not environment variables or config file), this information should be scrubbed from the archive in the `cmdline` file. +The `fleetctl debug archive` command retrieves information generated by Go's [`net/http/pprof`](https://golang.org/pkg/net/http/pprof/) package. In most scenarios this should not include sensitive information, however it does include command line arguments to the Fleet server. If the Fleet server receives sensitive credentials via CLI argument (not environment variables or config file), this information should be scrubbed from the archive in the `cmdline` file. From 4ace3b4bc5689557fcdfa01771a2b62a5659809f Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Thu, 23 Sep 2021 13:07:17 -0400 Subject: [PATCH 65/82] Fix Bug: Global maintainer can now edit SQL in create new query (#2204) --- changes/issue-2201-global-maintainer-create-query | 1 + frontend/components/forms/queries/QueryForm/QueryForm.tsx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/issue-2201-global-maintainer-create-query diff --git a/changes/issue-2201-global-maintainer-create-query b/changes/issue-2201-global-maintainer-create-query new file mode 100644 index 0000000000..4510cd6c41 --- /dev/null +++ b/changes/issue-2201-global-maintainer-create-query @@ -0,0 +1 @@ +- Bug fix: Global maintainer can create and save a new query \ No newline at end of file diff --git a/frontend/components/forms/queries/QueryForm/QueryForm.tsx b/frontend/components/forms/queries/QueryForm/QueryForm.tsx index e6b28ec340..7bf58d4528 100644 --- a/frontend/components/forms/queries/QueryForm/QueryForm.tsx +++ b/frontend/components/forms/queries/QueryForm/QueryForm.tsx @@ -434,7 +434,7 @@ const QueryForm = ({ }); } - if (isAnyTeamMaintainer || isGlobalMaintainer) { + if (isAnyTeamMaintainer) { return renderRunForMaintainer({ nameText, descText, queryValue }); } From fc009406607c9de93681f58d35b4b30b6d1d1e67 Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Thu, 23 Sep 2021 13:09:43 -0400 Subject: [PATCH 66/82] Team Settings: Press enter submits to create team (#2195) --- changes/issue-2078-enter-submits-create-team | 1 + .../components/CreateTeamModal/CreateTeamModal.tsx | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 changes/issue-2078-enter-submits-create-team diff --git a/changes/issue-2078-enter-submits-create-team b/changes/issue-2078-enter-submits-create-team new file mode 100644 index 0000000000..48aba2bfe5 --- /dev/null +++ b/changes/issue-2078-enter-submits-create-team @@ -0,0 +1 @@ +- Fix create team modal to submit with Enter key \ No newline at end of file diff --git a/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx b/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx index 25c15dc3b8..499d103055 100644 --- a/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx +++ b/frontend/pages/admin/TeamManagementPage/components/CreateTeamModal/CreateTeamModal.tsx @@ -41,7 +41,7 @@ const CreateTeamModal = (props: ICreateTeamModalProps): JSX.Element => { return ( -
    + {

    -
    + {/* must use ternary for NaN */} + {teamId && allTeamsScheduledQueriesList.length > 0 ? ( + <> + + + +
    +

    Queries from the “All teams”
    schedule run on this team’s hosts.

    \ + " + } + /> +
    + + ) : null} + {showInheritedQueries && + renderAllTeamsTable( + teamId, + allTeamsScheduledQueriesList, + allTeamsScheduledQueriesError + )} {showScheduleEditorModal && ( void; + onEditScheduledQueryClick?: ( + selectedQuery: IGlobalScheduledQuery | ITeamScheduledQuery + ) => void; allScheduledQueriesList: IGlobalScheduledQuery[] | ITeamScheduledQuery[]; - toggleScheduleEditorModal: () => void; + toggleScheduleEditorModal?: () => void; teamId: number; + inheritedQueries?: boolean; } interface IRootState { entities: { @@ -47,6 +54,7 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { toggleScheduleEditorModal, onEditScheduledQueryClick, teamId, + inheritedQueries, } = props; const dispatch = useDispatch(); const { MANAGE_PACKS } = paths; @@ -92,10 +100,14 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { ): void => { switch (action) { case "edit": - onEditScheduledQueryClick(global_scheduled_query); + if (onEditScheduledQueryClick) { + onEditScheduledQueryClick(global_scheduled_query); + } break; default: - onRemoveScheduledQueryClick([global_scheduled_query.id]); + if (onRemoveScheduledQueryClick) { + onRemoveScheduledQueryClick([global_scheduled_query.id]); + } break; } }; @@ -123,6 +135,33 @@ const ScheduleListWrapper = (props: IScheduleListWrapperProps): JSX.Element => { [dispatch] ); + const loadingInheritedQueriesTableData = useSelector((state: IRootState) => { + return state.entities.global_scheduled_queries.isLoading; + }); + + if (inheritedQueries) { + const inheritedQueriesTableHeaders = generateInheritedQueriesTableHeaders(); + + return ( +
    + +
    + ); + } + return (
    { + return [ + { + title: "Query", + Header: "Query", + disableSortBy: true, + accessor: "query_name", + Cell: (cellProps: ICellProps): JSX.Element => ( + + ), + }, + { + title: "Frequency", + Header: "Frequency", + disableSortBy: true, + accessor: "interval", + Cell: (cellProps: ICellProps): JSX.Element => ( + + ), + }, + ]; +}; + const generateActionDropdownOptions = (): IDropdownOption[] => { const dropdownOptions = [ { @@ -162,4 +185,8 @@ const generateDataSet = ( return [...enhanceAllScheduledQueryData(all_scheduled_queries, teamId)]; }; -export { generateTableHeaders, generateDataSet }; +export { + generateInheritedQueriesTableHeaders, + generateTableHeaders, + generateDataSet, +}; From 3987cdf82d7f8d55a3cd18344a428bf7ad8c34cb Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Thu, 23 Sep 2021 22:57:38 -0400 Subject: [PATCH 68/82] update deploy docs to mention the need for the mysql user to have the ability to create temporary tables for migrations (#2209) --- docs/02-Deploying/FAQ.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/02-Deploying/FAQ.md b/docs/02-Deploying/FAQ.md index f9580cbb68..84f8628337 100644 --- a/docs/02-Deploying/FAQ.md +++ b/docs/02-Deploying/FAQ.md @@ -12,6 +12,7 @@ - [Why am I receiving a database connection error when attempting to "prepare" the database?](#why-am-i-receiving-a-database-connection-error-when-attempting-to-prepare-the-database) - [Is Fleet available as a SaaS product?](#is-fleet-available-as-a-saas-product) - [Is Fleet compatible with X flavor of MySQL?](#is-fleet-compatible-with-x-flavor-of-mysql) +- [What are the MySQL user access requirements?](#what-are-the-mysql-user-requirements) ## How do I get support for working with Fleet? @@ -104,3 +105,7 @@ No. Currently, Fleet is only available for self-hosting on premises or in the cl ## Is Fleet compatible with X flavor of MySQL? Fleet is built to run on MySQL 5.7 or above. However, particularly with AWS Aurora, we recommend 2.10.0 and above, as we've seen issues with anything below that. + +## What are the MySQL user requirements? + +The user `fleet prepare db` (via environment variable `FLEET_MYSQL_USERNAME` or command line flag `--mysql_username=`) uses to interact with the database needs to be able to create, alter, and drop tables as well as the ability to create temporary tables. \ No newline at end of file From 655b57789d9d392e31c977a45f49f5901e8a98b8 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Thu, 23 Sep 2021 23:38:21 -0400 Subject: [PATCH 69/82] add faq about host team enrollment --- docs/01-Using-Fleet/FAQ.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/01-Using-Fleet/FAQ.md b/docs/01-Using-Fleet/FAQ.md index 62771ea3e3..bc334c2733 100644 --- a/docs/01-Using-Fleet/FAQ.md +++ b/docs/01-Using-Fleet/FAQ.md @@ -13,6 +13,7 @@ - [Why aren’t my live queries being logged?](#why-arent-my-live-queries-being-logged) - [Can I use the Fleet API to fetch results from a scheduled query pack?](#can-i-use-the-fleet-api-to-fetch-results-from-a-scheduled-query-pack) - [How do I automatically add hosts to packs when the hosts enroll to Fleet?](#how-do-i-automatically-add-hosts-to-packs-when-the-hosts-enroll-to-Fleet) +- [How do I automatically assign a host to a team when it enrolls with Fleet?](#how-do-i-automatically-assign-a-host-to-a-team-when-it-enrolls-with-fleet) - [How do I resolve an "unknown column" error when upgrading Fleet?](#how-do-i-resolve-an-unknown-column-error-when-upgrading-fleet) ## What do I need to do to switch from Kolide Fleet to FleetDM Fleet? @@ -124,6 +125,10 @@ When your hosts enroll to Fleet, they will become a member of the label and, bec You can also do this by setting the `targets` field in the [YAML configuration file](./02-fleetctl-CLI.md#query-packs) that manages the packs that are added to your Fleet instance. +## How do I automatically assign a host to a team when it enrolls with Fleet? + +[Team Enroll Secrets](https://github.com/fleetdm/fleet/blob/main/docs/01-Using-Fleet/10-Teams.md#enroll-hosts-to-a-team) allow you to automatically assign a host to a team. + ## How do I resolve an "unknown column" error when upgrading Fleet? The `unknown column` error typically occurs when the database migrations haven't been run during the upgrade process. From d81a6317a005b868c84a3970bff57eb15fca35d7 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Fri, 24 Sep 2021 15:56:55 -0300 Subject: [PATCH 70/82] Return host count when modifying a label (#2221) --- changes/issue-1984-label-count-on-edit | 1 + server/datastore/mysql/labels.go | 22 ++++++++++++---------- server/datastore/mysql/labels_test.go | 23 +++++++++++++++++++---- 3 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 changes/issue-1984-label-count-on-edit diff --git a/changes/issue-1984-label-count-on-edit b/changes/issue-1984-label-count-on-edit new file mode 100644 index 0000000000..44e8a302d8 --- /dev/null +++ b/changes/issue-1984-label-count-on-edit @@ -0,0 +1 @@ +* Do not reset label counts when modifying them. diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index ca5482e033..9b46ac0714 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -217,17 +217,12 @@ func (d *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...fl } func (d *Datastore) SaveLabel(ctx context.Context, label *fleet.Label) (*fleet.Label, error) { - query := ` - UPDATE labels SET - name = ?, - description = ? - WHERE id = ? - ` + query := `UPDATE labels SET name = ?, description = ? WHERE id = ?` _, err := d.writer.ExecContext(ctx, query, label.Name, label.Description, label.ID) if err != nil { return nil, errors.Wrap(err, "saving label") } - return label, nil + return labelDB(ctx, label.ID, d.writer) } // DeleteLabel deletes a fleet.Label @@ -237,13 +232,20 @@ func (d *Datastore) DeleteLabel(ctx context.Context, name string) error { // Label returns a fleet.Label identified by lid if one exists. func (d *Datastore) Label(ctx context.Context, lid uint) (*fleet.Label, error) { + return labelDB(ctx, lid, d.reader) +} + +func labelDB(ctx context.Context, lid uint, q sqlx.QueryerContext) (*fleet.Label, error) { sql := ` - SELECT * FROM labels - WHERE id = ? + SELECT + l.*, + (SELECT COUNT(1) FROM label_membership lm JOIN hosts h ON (lm.host_id = h.id) WHERE label_id = l.id) AS host_count + FROM labels l + WHERE id = ? ` label := &fleet.Label{} - if err := sqlx.GetContext(ctx, d.reader, label, sql, lid); err != nil { + if err := sqlx.GetContext(ctx, q, label, sql, lid); err != nil { return nil, errors.Wrap(err, "selecting label") } diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index d1a7d77b71..3aa2ee7f8b 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -594,22 +594,37 @@ func testLabelsIDsByName(t *testing.T, ds *Datastore) { } func testLabelsSave(t *testing.T, db *Datastore) { + h1, err := db.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + SeenTime: time.Now(), + OsqueryHostID: "1", + NodeKey: "1", + UUID: "1", + Hostname: "foo.local", + }) + require.NoError(t, err) + label := &fleet.Label{ Name: "my label", Description: "a label", Query: "select 1 from processes;", Platform: "darwin", } - label, err := db.NewLabel(context.Background(), label) - require.Nil(t, err) + label, err = db.NewLabel(context.Background(), label) + require.NoError(t, err) label.Name = "changed name" label.Description = "changed description" + + require.NoError(t, db.RecordLabelQueryExecutions(context.Background(), h1, map[uint]*bool{label.ID: ptr.Bool(true)}, time.Now())) + _, err = db.SaveLabel(context.Background(), label) - require.Nil(t, err) + require.NoError(t, err) saved, err := db.Label(context.Background(), label.ID) - require.Nil(t, err) + require.NoError(t, err) assert.Equal(t, label.Name, saved.Name) assert.Equal(t, label.Description, saved.Description) + assert.Equal(t, 1, saved.HostCount) } func testLabelsQueriesForCentOSHost(t *testing.T, db *Datastore) { From fc19722ec29641fd4abb1474ea7cf49a6fca4873 Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Fri, 24 Sep 2021 15:09:15 -0400 Subject: [PATCH 71/82] Sentence case for Select targets, All hosts, Linux (#2219) --- .../QueryPage/screens/SelectTargets.tsx | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/frontend/pages/queries/QueryPage/screens/SelectTargets.tsx b/frontend/pages/queries/QueryPage/screens/SelectTargets.tsx index cafa9d629f..c5b4997efa 100644 --- a/frontend/pages/queries/QueryPage/screens/SelectTargets.tsx +++ b/frontend/pages/queries/QueryPage/screens/SelectTargets.tsx @@ -47,17 +47,30 @@ const TargetPillSelector = ({ entity, isSelected, onClick, -}: ITargetPillSelectorProps): JSX.Element => ( - -); +}: ITargetPillSelectorProps): JSX.Element => { + const displayText = () => { + switch (entity.display_text) { + case "All Hosts": + return "All hosts"; + case "All Linux": + return "Linux"; + default: + return entity.display_text; + } + }; + + return ( + + ); +}; const SelectTargets = ({ baseClass, @@ -228,7 +241,7 @@ const SelectTargets = ({ if (isEmpty(searchText) && isTargetsLoading) { return (
    -

    Select Targets

    +

    Select targets

    @@ -239,7 +252,7 @@ const SelectTargets = ({ if (isEmpty(searchText) && isTargetsError) { return (
    -

    Select Targets

    +

    Select targets

    @@ -264,7 +277,7 @@ const SelectTargets = ({ return (
    -

    Select Targets

    +

    Select targets

    {allHostsLabels && allHostsLabels.length > 0 && From df89added96e6df2b108b8d7848a7216b2925896 Mon Sep 17 00:00:00 2001 From: Zach Wasserman Date: Sun, 26 Sep 2021 16:35:01 -0700 Subject: [PATCH 72/82] Update pull request template (#2234) --- .github/pull_request_template.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index cc86dd9724..d8297eaafa 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,9 @@ # Checklist for submitter -If some of the following don't apply, please write a short explanation why. +If some of the following don't apply, delete the relevant line. -- [ ] Changes file added (if needed) +- [ ] Changes file added (for user-visible changes) - [ ] Documented any API changes - [ ] Documented any permissions changes -- [ ] Added tests for all functionality -- [ ] Manual QA for all functionality +- [ ] Added/updated tests +- [ ] Manual QA for all new/changed functionality From 429875d4e547618f8ac289c433c480d50dbf7add Mon Sep 17 00:00:00 2001 From: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> Date: Mon, 27 Sep 2021 20:25:23 +0900 Subject: [PATCH 73/82] Update 00-Learn-how-to-use-Fleet.md (#2217) Updated to reflect latest UI and query library content changes in Fleet 4.3.0. --- docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md b/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md index aa7a894d83..c10538fcc9 100644 --- a/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md +++ b/docs/01-Using-Fleet/00-Learn-how-to-use-Fleet.md @@ -28,19 +28,14 @@ So, let's start by asking the following questions about Fleet's 7 simulated Linu 2. Do these devices have a high severity vulnerable version of OpenSSL installed? -These questions can easily be answered, by running the following query: "Detect Linux hosts with high severity vulnerable versions of OpenSSL." +These questions can easily be answered, by running this simple query: "Get OpenSSL versions." -On the **Queries** page, enter the query name, "Detect Linux hosts with high severity vulnerable versions of OpenSSL," in the search box, select the query from the results table, and navigate to the **Edit or run query** page. +On the **Queries** page, enter the query name, "Get OpenSSL versions," in the search box, and select it to enter the **query console**. Then from the **query console**, hit "Run query", and from the "Select targets" page, select "All hosts," to run this query against all hosts enrolled in your Fleet. Then hit the "Run" button to execute the query. -Fleet query search +Fleet select targets -On the **Edit or run query** page, open the "Select targets" dropdown, and press the purple "+" icon to the right of "All hosts," to run this query against all hosts enrolled in your Fleet. Then hit the "Run" button to execute the query. - - -Fleet select targets - The query may take several seconds to complete, because Fleet has to wait for the osquery agents to respond with results. > Fleet's query response time is inherently variable because of osquery's heartbeat response time. This helps prevent performance issues on hosts. @@ -48,7 +43,7 @@ The query may take several seconds to complete, because Fleet has to wait for th When the query has finished, you should see 4 columns and several rows in the "Results" table: -Fleet query results +Fleet query results - The "hostname" column answers: which device responded for a given row of results? From 97750d1e074aa89f6e39f92f308506b1d2fc556e Mon Sep 17 00:00:00 2001 From: noahtalerman <47070608+noahtalerman@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:08:40 -0400 Subject: [PATCH 74/82] Improve examples and API documentation for managing teams with `fleetctl` (#2199) - Add example `team.yml` configuration file. A file with this format can be used to apply teams using `fleetctl apply` - Add `spec/teams` API route to API docs --- docs/01-Using-Fleet/03-REST-API.md | 68 +++++++++++++++++++ .../multi-file-configuration/team.yml | 24 +++++++ 2 files changed, 92 insertions(+) create mode 100644 docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml diff --git a/docs/01-Using-Fleet/03-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md index dfe6d99ab7..3418904b96 100644 --- a/docs/01-Using-Fleet/03-REST-API.md +++ b/docs/01-Using-Fleet/03-REST-API.md @@ -6066,6 +6066,74 @@ _Available in Fleet Premium_ `Status: 200` +### Apply team spec + +_Available in Fleet Premium_ + +If the `name` specified is associated with an existing team, this API route, completely replaces this team's existing `agent_options` and `secrets` with those that are specified. + +If the `name` is not already associated with an existing team, this API route creates a new team with the specified `name`, `agent_options`, and `secrets`. + +`POST /api/v1/fleet/spec/teams` + +#### Parameters + +| Name | Type | In | Description | +| ---- | ------ | ---- | ------------------------------ | +| name | string | body | **Required.** The team's name. | +| agent_options | string | body | **Required.** The agent options spec that is applied to the hosts assigned to the specified to team. These agent agent options completely override the global agent options specified in the [`GET /api/v1/fleet/config API route`](#get-configuration)| +| secrets | list | body | **Required.** A list of plain text strings used as the enroll secrets. | + +#### Example + +`POST /api/v1/fleet/spec/teams` + +##### Request body + +```json +{ + "specs": [ + { + "name": "Client Platform Engineering", + "agent_options": { + "spec": { + "config": { + "options": { + "logger_plugin": "tls", + "pack_delimiter": "/", + "logger_tls_period": 10, + "distributed_plugin": "tls", + "disable_distributed": false, + "logger_tls_endpoint": "/api/v1/osquery/log", + "distributed_interval": 10, + "distributed_tls_max_attempts": 3 + }, + "decorators": { + "load": [ + "SELECT uuid AS host_uuid FROM system_info;", + "SELECT hostname AS hostname FROM system_info;" + ] + } + }, + "overrides": {} + } + }, + "secrets": [ + { + "secret": "fTp52/twaxBU6gIi0J6PHp8o5Sm1k1kn", + }, + { + "secret": "bhD5kiX2J+KBgZSk118qO61ZIdX/v8On", + } + ] + } + ] +} +``` + +#### Default response + +`Status: 200` --- diff --git a/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml new file mode 100644 index 0000000000..a85702f19e --- /dev/null +++ b/docs/01-Using-Fleet/configuration-files/multi-file-configuration/team.yml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: team +spec: + team: + name: Client Platform Engineerin + agent_options: + config: + decorators: + load: + - SELECT uuid AS host_uuid FROM system_info; + - SELECT hostname AS hostname FROM system_info; + options: + disable_distributed: false + distributed_interval: 10 + distributed_plugin: tls + distributed_tls_max_attempts: 3 + logger_plugin: tls + logger_tls_endpoint: /api/v1/osquery/log + logger_tls_period: 10 + pack_delimiter: / + overrides: {} + secrets: + - secret: RzTlxPvugG4o4O5IKS/HqEDJUmI1hwBoffff From 433c76af04cfd71955e07665c24f0bd34664f01a Mon Sep 17 00:00:00 2001 From: gillespi314 <73313222+gillespi314@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:28:03 -0500 Subject: [PATCH 75/82] Fix non-unique key warnings on user table (#2223) --- frontend/pages/hosts/HostDetailsPage/HostDetailsPage.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.jsx b/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.jsx index 80ab294597..26228754bc 100644 --- a/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.jsx +++ b/frontend/pages/hosts/HostDetailsPage/HostDetailsPage.jsx @@ -493,7 +493,7 @@ export class HostDetailsPage extends Component { {users.map((hostUser) => { return ( ); From 5653f1e8686e6eb27bd6fe775ec88b314f4f1950 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 27 Sep 2021 14:02:11 -0300 Subject: [PATCH 76/82] Update URLs from team to teams, add tests for policy auth (#2228) * Update URLs from team to teams, add tests for policy auth * Fix test * Address review comments --- ...160-straighten-permissions-around-policies | 1 + docs/01-Using-Fleet/03-REST-API.md | 32 +++--- docs/01-Using-Fleet/09-Permissions.md | 5 +- frontend/fleet/endpoints.ts | 2 +- .../test/mocks/team_scheduled_query_mocks.js | 8 +- server/authz/authz.go | 4 + server/authz/policy.rego | 23 ++++ server/fleet/global_policies.go | 4 +- server/service/global_policies_test.go | 84 ++++++++++++++ server/service/handler.go | 14 +++ server/service/integration_enterprise_test.go | 26 ++--- server/service/team_policies.go | 9 +- server/service/team_policies_test.go | 107 ++++++++++++++++++ 13 files changed, 278 insertions(+), 41 deletions(-) create mode 100644 changes/issue-2160-straighten-permissions-around-policies create mode 100644 server/service/global_policies_test.go create mode 100644 server/service/team_policies_test.go diff --git a/changes/issue-2160-straighten-permissions-around-policies b/changes/issue-2160-straighten-permissions-around-policies new file mode 100644 index 0000000000..370c57ad9b --- /dev/null +++ b/changes/issue-2160-straighten-permissions-around-policies @@ -0,0 +1 @@ +* Expand and improve how roles interact with policies. diff --git a/docs/01-Using-Fleet/03-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md index 3418904b96..9d0629e1d8 100644 --- a/docs/01-Using-Fleet/03-REST-API.md +++ b/docs/01-Using-Fleet/03-REST-API.md @@ -3212,7 +3212,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Get team schedule -`GET /api/v1/fleet/team/{id}/schedule` +`GET /api/v1/fleet/teams/{id}/schedule` #### Parameters @@ -3226,7 +3226,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`GET /api/v1/fleet/team/2/schedule` +`GET /api/v1/fleet/teams/2/schedule` ##### Default response @@ -3275,7 +3275,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Add query to team schedule -`POST /api/v1/fleet/team/{id}/schedule` +`POST /api/v1/fleet/teams/{id}/schedule` #### Parameters @@ -3292,7 +3292,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`POST /api/v1/fleet/team/2/schedule` +`POST /api/v1/fleet/teams/2/schedule` ##### Request body @@ -3330,7 +3330,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Edit query in team schedule -`PATCH /api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}` +`PATCH /api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}` #### Parameters @@ -3347,7 +3347,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`PATCH /api/v1/fleet/team/2/schedule/5` +`PATCH /api/v1/fleet/teams/2/schedule/5` ##### Request body @@ -3384,7 +3384,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Remove query from team schedule -`DELETE /api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}` +`DELETE /api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}` #### Parameters @@ -3395,7 +3395,7 @@ This allows you to easily configure scheduled queries that will impact a whole t #### Example -`DELETE /api/v1/fleet/team/2/schedule/5` +`DELETE /api/v1/fleet/teams/2/schedule/5` ##### Default response @@ -4384,7 +4384,7 @@ Team policies work the same as policies, but at the team level. ### List team policies -`GET /api/v1/fleet/team/{team_id}/policies` +`GET /api/v1/fleet/teams/{team_id}/policies` #### Parameters @@ -4394,7 +4394,7 @@ Team policies work the same as policies, but at the team level. #### Example -`GET /api/v1/fleet/team/1/policies` +`GET /api/v1/fleet/teams/1/policies` ##### Default response @@ -4423,7 +4423,7 @@ Team policies work the same as policies, but at the team level. ### Get team policy by ID -`GET /api/v1/fleet/team/{team_id}/policies/{id}` +`GET /api/v1/fleet/teams/{team_id}/policies/{id}` #### Parameters @@ -4434,7 +4434,7 @@ Team policies work the same as policies, but at the team level. #### Example -`GET /api/v1/fleet/team/1/policies/1` +`GET /api/v1/fleet/teams/1/policies/1` ##### Default response @@ -4454,7 +4454,7 @@ Team policies work the same as policies, but at the team level. ### Add team policy -`POST /api/v1/fleet/team/{team_id}/policies` +`POST /api/v1/fleet/teams/{team_id}/policies` #### Parameters @@ -4465,7 +4465,7 @@ Team policies work the same as policies, but at the team level. #### Example -`POST /api/v1/fleet/team/1/policies` +`POST /api/v1/fleet/teams/1/policies` #### Request body @@ -4493,7 +4493,7 @@ Team policies work the same as policies, but at the team level. ### Remove team policies -`POST /api/v1/fleet/team/{team_id}/policies/delete` +`POST /api/v1/fleet/teams/{team_id}/policies/delete` #### Parameters @@ -4504,7 +4504,7 @@ Team policies work the same as policies, but at the team level. #### Example -`POST /api/v1/fleet/global/policies/delete` +`POST /api/v1/fleet/teams/1/policies/delete` #### Request body diff --git a/docs/01-Using-Fleet/09-Permissions.md b/docs/01-Using-Fleet/09-Permissions.md index 0795a57348..17e2acba77 100644 --- a/docs/01-Using-Fleet/09-Permissions.md +++ b/docs/01-Using-Fleet/09-Permissions.md @@ -35,6 +35,8 @@ The following table depicts various permissions levels for each role. | Create labels | | ✅ | ✅ | | Edit labels | | ✅ | ✅ | | Delete labels | | ✅ | ✅ | +| Create new global policies | | ✅ | ✅ | +| Delete global policies | | ✅ | ✅ | | Create users | | | ✅ | | Edit users | | | ✅ | | Delete users | | | ✅ | @@ -72,8 +74,9 @@ The following table depicts various permissions levels in a team. | Filter hosts assigned to team using policies | ✅ | ✅ | | Filter hosts assigned to team using labels | ✅ | ✅ | | Target hosts assigned to team using labels | ✅ | ✅ | -| Browse policies for hosts assigned to team | ✅ | ✅ | | Run saved queries as live queries on hosts assigned to team | ✅ | ✅ | +| Create new team policies | | ✅ | +| Delete team policies | | ✅ | | Run custom queries as live queries on hosts assigned to team | | ✅ | | Enroll hosts to member team | | ✅ | | Delete hosts belonging to member team | | ✅ | diff --git a/frontend/fleet/endpoints.ts b/frontend/fleet/endpoints.ts index 13267fbd1c..6b58583227 100644 --- a/frontend/fleet/endpoints.ts +++ b/frontend/fleet/endpoints.ts @@ -41,7 +41,7 @@ export default { STATUS_LABEL_COUNTS: "/v1/fleet/host_summary", TARGETS: "/v1/fleet/targets", TEAM_SCHEDULE: (id: number): string => { - return `/v1/fleet/team/${id}/schedule`; + return `/v1/fleet/teams/${id}/schedule`; }, USERS: "/v1/fleet/users", USERS_ADMIN: "/v1/fleet/users/admin", diff --git a/frontend/test/mocks/team_scheduled_query_mocks.js b/frontend/test/mocks/team_scheduled_query_mocks.js index 30a5533a34..bfbe3e7d1a 100644 --- a/frontend/test/mocks/team_scheduled_query_mocks.js +++ b/frontend/test/mocks/team_scheduled_query_mocks.js @@ -16,7 +16,7 @@ export default { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${params.team_id}/schedule`, + endpoint: `/api/v1/fleet/teams/${params.team_id}/schedule`, method: "post", params, response: { scheduled: teamScheduledQueryStub }, @@ -28,7 +28,7 @@ export default { valid: (bearerToken, teamScheduledQuery) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, + endpoint: `/api/v1/fleet/teams/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, method: "delete", response: {}, }); @@ -38,7 +38,7 @@ export default { valid: (bearerToken, teamID) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${teamID}/schedule`, + endpoint: `/api/v1/fleet/teams/${teamID}/schedule`, method: "get", response: { scheduled: [teamScheduledQueryStub] }, }); @@ -48,7 +48,7 @@ export default { valid: (bearerToken, teamScheduledQuery, params) => { return createRequestMock({ bearerToken, - endpoint: `/api/v1/fleet/team/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, + endpoint: `/api/v1/fleet/teams/${teamScheduledQuery.team_id}/schedule/${teamScheduledQuery.id}`, method: "patch", params, response: { scheduled: { ...teamScheduledQuery, ...params } }, diff --git a/server/authz/authz.go b/server/authz/authz.go index 6402512253..35d02b96ac 100644 --- a/server/authz/authz.go +++ b/server/authz/authz.go @@ -130,6 +130,10 @@ func (a *Authorizer) TeamAuthorize(ctx context.Context, teamID uint, action stri switch *subject.GlobalRole { case fleet.RoleAdmin, fleet.RoleMaintainer: return nil + case fleet.RoleObserver: + if action == fleet.ActionRead { + return nil + } } } diff --git a/server/authz/policy.rego b/server/authz/policy.rego index 64ccb76a09..ff9a5ecf8a 100644 --- a/server/authz/policy.rego +++ b/server/authz/policy.rego @@ -369,18 +369,41 @@ allow { } allow { + is_null(object.team_id) object.type == "policy" subject.global_role == maintainer action == [read, write][_] } +allow { + object.type == "policy" + subject.global_role == maintainer + action == [read][_] +} + +# Global Observer users can read policies +allow { + object.type == "policy" + subject.global_role == observer + action == [read][_] +} + # Team Maintainers can read and write policies allow { + not is_null(object.team_id) object.type == "policy" team_role(subject, subject.teams[_].id) == maintainer action == [read, write][_] } +# Team Observer can read policies +allow { + not is_null(object.team_id) + object.type == "policy" + team_role(subject, subject.teams[_].id) == observer + action == [read][_] +} + ## # Software ## diff --git a/server/fleet/global_policies.go b/server/fleet/global_policies.go index d84f594c45..7034f0fd2b 100644 --- a/server/fleet/global_policies.go +++ b/server/fleet/global_policies.go @@ -6,11 +6,11 @@ type Policy struct { QueryName string `json:"query_name" db:"query_name"` PassingHostCount uint `json:"passing_host_count" db:"passing_host_count"` FailingHostCount uint `json:"failing_host_count" db:"failing_host_count"` - TeamID *uint `db:"team_id"` + TeamID *uint `json:"team_id" db:"team_id"` UpdateCreateTimestamps } -func (Policy) AuthzType() string { +func (p Policy) AuthzType() string { return "policy" } diff --git a/server/service/global_policies_test.go b/server/service/global_policies_test.go new file mode 100644 index 0000000000..284cccb4eb --- /dev/null +++ b/server/service/global_policies_test.go @@ -0,0 +1,84 @@ +package service + +import ( + "context" + "testing" + + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" +) + +func TestGlobalPoliciesAuth(t *testing.T) { + ds := new(mock.Store) + svc := newTestService(ds, nil, nil) + + ds.NewGlobalPolicyFunc = func(ctx context.Context, queryID uint) (*fleet.Policy, error) { + return nil, nil + } + ds.ListGlobalPoliciesFunc = func(ctx context.Context) ([]*fleet.Policy, error) { + return nil, nil + } + ds.PolicyFunc = func(ctx context.Context, id uint) (*fleet.Policy, error) { + return nil, nil + } + ds.DeleteGlobalPoliciesFunc = func(ctx context.Context, ids []uint) ([]uint, error) { + return nil, nil + } + + var testCases = []struct { + name string + user *fleet.User + shouldFailWrite bool + shouldFailRead bool + }{ + { + "global admin", + &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, + false, + false, + }, + { + "global maintainer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)}, + false, + false, + }, + { + "global observer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, + true, + false, + }, + { + "team maintainer", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}}, + true, + true, + }, + { + "team observer, belongs to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}}, + true, + true, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ctx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) + + _, err := svc.NewGlobalPolicy(ctx, 2) + checkAuthErr(t, tt.shouldFailWrite, err) + + _, err = svc.ListGlobalPolicies(ctx) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.GetPolicyByIDQueries(ctx, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.DeleteGlobalPolicies(ctx, []uint{1}) + checkAuthErr(t, tt.shouldFailWrite, err) + }) + } +} diff --git a/server/service/handler.go b/server/service/handler.go index f6c894977c..edbb178d86 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -695,20 +695,34 @@ func attachNewStyleFleetAPIRoutes(r *mux.Router, svc fleet.Service, opts []kitht e.POST("/api/v1/fleet/translate", translatorEndpoint, translatorRequest{}) e.POST("/api/v1/fleet/spec/teams", applyTeamSpecsEndpoint, applyTeamSpecsRequest{}) + // Alias /api/v1/fleet/team/ -> /api/v1/fleet/teams/ e.GET("/api/v1/fleet/team/{team_id}/schedule", getTeamScheduleEndpoint, getTeamScheduleRequest{}) e.POST("/api/v1/fleet/team/{team_id}/schedule", teamScheduleQueryEndpoint, teamScheduleQueryRequest{}) e.PATCH("/api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}", modifyTeamScheduleEndpoint, modifyTeamScheduleRequest{}) e.DELETE("/api/v1/fleet/team/{team_id}/schedule/{scheduled_query_id}", deleteTeamScheduleEndpoint, deleteTeamScheduleRequest{}) + // End alias + + e.GET("/api/v1/fleet/teams/{team_id}/schedule", getTeamScheduleEndpoint, getTeamScheduleRequest{}) + e.POST("/api/v1/fleet/teams/{team_id}/schedule", teamScheduleQueryEndpoint, teamScheduleQueryRequest{}) + e.PATCH("/api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}", modifyTeamScheduleEndpoint, modifyTeamScheduleRequest{}) + e.DELETE("/api/v1/fleet/teams/{team_id}/schedule/{scheduled_query_id}", deleteTeamScheduleEndpoint, deleteTeamScheduleRequest{}) e.POST("/api/v1/fleet/global/policies", globalPolicyEndpoint, globalPolicyRequest{}) e.GET("/api/v1/fleet/global/policies", listGlobalPoliciesEndpoint, nil) e.GET("/api/v1/fleet/global/policies/{policy_id}", getPolicyByIDEndpoint, getPolicyByIDRequest{}) e.POST("/api/v1/fleet/global/policies/delete", deleteGlobalPoliciesEndpoint, deleteGlobalPoliciesRequest{}) + // Alias /api/v1/fleet/team/ -> /api/v1/fleet/teams/ e.POST("/api/v1/fleet/team/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) e.GET("/api/v1/fleet/team/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) e.GET("/api/v1/fleet/team/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) e.POST("/api/v1/fleet/team/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) + // End alias + + e.POST("/api/v1/fleet/teams/{team_id}/policies", teamPolicyEndpoint, teamPolicyRequest{}) + e.GET("/api/v1/fleet/teams/{team_id}/policies", listTeamPoliciesEndpoint, listTeamPoliciesRequest{}) + e.GET("/api/v1/fleet/teams/{team_id}/policies/{policy_id}", getTeamPolicyByIDEndpoint, getTeamPolicyByIDRequest{}) + e.POST("/api/v1/fleet/teams/{team_id}/policies/delete", deleteTeamPoliciesEndpoint, deleteTeamPoliciesRequest{}) e.GET("/api/v1/fleet/software", listSoftwareEndpoint, listSoftwareRequest{}) } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index a6b7ea9b14..5370f50b82 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -103,7 +103,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { require.NoError(t, err) ts := getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 0) qr, err := s.ds.NewQuery( @@ -114,10 +114,10 @@ func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { gsParams := teamScheduleQueryRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{QueryID: &qr.ID, Interval: ptr.Uint(42)}} r := teamScheduleQueryResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), gsParams, http.StatusOK, &r) + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), gsParams, http.StatusOK, &r) ts = getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 1) assert.Equal(t, uint(42), ts.Scheduled[0].Interval) assert.Equal(t, "TestQueryTeamPolicy", ts.Scheduled[0].Name) @@ -126,21 +126,21 @@ func (s *integrationEnterpriseTestSuite) TestTeamSchedule() { modifyResp := modifyTeamScheduleResponse{} modifyParams := modifyTeamScheduleRequest{ScheduledQueryPayload: fleet.ScheduledQueryPayload{Interval: ptr.Uint(55)}} - s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), modifyParams, http.StatusOK, &modifyResp) + s.DoJSON("PATCH", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), modifyParams, http.StatusOK, &modifyResp) // just to satisfy my paranoia, wanted to make sure the contents of the json would work - s.DoRaw("PATCH", fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), []byte(`{"interval": 77}`), http.StatusOK) + s.DoRaw("PATCH", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), []byte(`{"interval": 77}`), http.StatusOK) ts = getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 1) assert.Equal(t, uint(77), ts.Scheduled[0].Interval) deleteResp := deleteTeamScheduleResponse{} - s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/team/%d/schedule/%d", team1.ID, id), nil, http.StatusOK, &deleteResp) + s.DoJSON("DELETE", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule/%d", team1.ID, id), nil, http.StatusOK, &deleteResp) ts = getTeamScheduleResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/schedule", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/schedule", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Scheduled, 0) } @@ -180,7 +180,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicies() { s.token = s.getTestToken(email, password) ts := listTeamPoliciesResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Policies, 0) qr, err := s.ds.NewQuery(context.Background(), &fleet.Query{Name: "TestQuery2", Description: "Some description", Query: "select * from osquery;", ObserverCanRun: true}) @@ -188,19 +188,19 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicies() { tpParams := teamPolicyRequest{QueryID: qr.ID} r := teamPolicyResponse{} - s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), tpParams, http.StatusOK, &r) + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), tpParams, http.StatusOK, &r) ts = listTeamPoliciesResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Policies, 1) assert.Equal(t, "TestQuery2", ts.Policies[0].QueryName) assert.Equal(t, qr.ID, ts.Policies[0].QueryID) deletePolicyParams := deleteTeamPoliciesRequest{IDs: []uint{ts.Policies[0].ID}} deletePolicyResp := deleteTeamPoliciesResponse{} - s.DoJSON("POST", "/api/v1/fleet/global/policies/delete", deletePolicyParams, http.StatusOK, &deletePolicyResp) + s.DoJSON("POST", fmt.Sprintf("/api/v1/fleet/teams/%d/policies/delete", team1.ID), deletePolicyParams, http.StatusOK, &deletePolicyResp) ts = listTeamPoliciesResponse{} - s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/team/%d/policies", team1.ID), nil, http.StatusOK, &ts) + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) require.Len(t, ts.Policies, 0) } diff --git a/server/service/team_policies.go b/server/service/team_policies.go index 3ac3822345..9ea4febc28 100644 --- a/server/service/team_policies.go +++ b/server/service/team_policies.go @@ -4,6 +4,7 @@ import ( "context" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" ) ///////////////////////////////////////////////////////////////////////////////// @@ -32,7 +33,7 @@ func teamPolicyEndpoint(ctx context.Context, request interface{}, svc fleet.Serv } func (svc Service) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionWrite); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionWrite); err != nil { return nil, err } if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { @@ -67,7 +68,7 @@ func listTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc flee } func (svc Service) ListTeamPolicies(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionRead); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionRead); err != nil { return nil, err } if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { @@ -103,7 +104,7 @@ func getTeamPolicyByIDEndpoint(ctx context.Context, request interface{}, svc fle } func (svc Service) GetTeamPolicyByIDQueries(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionRead); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionRead); err != nil { return nil, err } if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionRead); err != nil { @@ -144,7 +145,7 @@ func deleteTeamPoliciesEndpoint(ctx context.Context, request interface{}, svc fl } func (svc Service) DeleteTeamPolicies(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { - if err := svc.authz.Authorize(ctx, &fleet.Policy{}, fleet.ActionWrite); err != nil { + if err := svc.authz.Authorize(ctx, &fleet.Policy{TeamID: ptr.Uint(teamID)}, fleet.ActionWrite); err != nil { return nil, err } if err := svc.authz.TeamAuthorize(ctx, teamID, fleet.ActionWrite); err != nil { diff --git a/server/service/team_policies_test.go b/server/service/team_policies_test.go new file mode 100644 index 0000000000..662dcce3ea --- /dev/null +++ b/server/service/team_policies_test.go @@ -0,0 +1,107 @@ +package service + +import ( + "context" + "testing" + + "github.com/fleetdm/fleet/v4/server/authz" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/require" +) + +func TestTeamPoliciesAuth(t *testing.T) { + ds := new(mock.Store) + svc := newTestService(ds, nil, nil) + + ds.NewTeamPolicyFunc = func(ctx context.Context, teamID uint, queryID uint) (*fleet.Policy, error) { + return &fleet.Policy{}, nil + } + ds.ListTeamPoliciesFunc = func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) { + return nil, nil + } + ds.TeamPolicyFunc = func(ctx context.Context, teamID uint, policyID uint) (*fleet.Policy, error) { + return nil, nil + } + ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, ids []uint) ([]uint, error) { + return nil, nil + } + + var testCases = []struct { + name string + user *fleet.User + shouldFailWrite bool + shouldFailRead bool + }{ + { + "global admin", + &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}, + false, + false, + }, + { + "global maintainer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)}, + true, + false, + }, + { + "global observer", + &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)}, + true, + false, + }, + { + "team maintainer, belongs to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}}, + false, + false, + }, + { + "team observer, belongs to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}}, + true, + false, + }, + { + "team maintainer, DOES NOT belong to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 2}, Role: fleet.RoleMaintainer}}}, + true, + true, + }, + { + "team observer, DOES NOT belong to team", + &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 2}, Role: fleet.RoleObserver}}}, + true, + true, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ctx := viewer.NewContext(context.Background(), viewer.Viewer{User: tt.user}) + + _, err := svc.NewTeamPolicy(ctx, 1, 2) + checkAuthErr(t, tt.shouldFailWrite, err) + + _, err = svc.ListTeamPolicies(ctx, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.GetTeamPolicyByIDQueries(ctx, 1, 1) + checkAuthErr(t, tt.shouldFailRead, err) + + _, err = svc.DeleteTeamPolicies(ctx, 1, []uint{1}) + checkAuthErr(t, tt.shouldFailWrite, err) + }) + } +} + +func checkAuthErr(t *testing.T, shouldFail bool, err error) { + if shouldFail { + require.Error(t, err) + require.Equal(t, (&authz.Forbidden{}).Error(), err.Error()) + } else { + require.NoError(t, err) + } +} From b63cf9d125e61be7eca44a4135a28b50dc6b1542 Mon Sep 17 00:00:00 2001 From: Renee Jackson <44620612+rlynnj11@users.noreply.github.com> Date: Mon, 27 Sep 2021 14:02:38 -0300 Subject: [PATCH 77/82] Create smoke-tests.md (#2237) Add issue template for per-release smoke tests. Increases visibility. --- .github/ISSUE_TEMPLATE/smoke-tests.md | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/smoke-tests.md diff --git a/.github/ISSUE_TEMPLATE/smoke-tests.md b/.github/ISSUE_TEMPLATE/smoke-tests.md new file mode 100644 index 0000000000..fb9d454adf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/smoke-tests.md @@ -0,0 +1,67 @@ +--- +name: Release QA +about: Checklist of required tests prior to release +title: '' +labels: '' +assignees: '' + +--- + +# Goal: easy-to-follow test steps for sanity checking a release manually + +**Fleet version** (Head to the "My account" page in the Fleet UI or run `fleetctl version`): + +**Web browser** _(e.g. Chrome 88.0.4324)_: + +# Important reference data + +1. [fleetctl preview setup](https://fleetdm.com/get-started) +2. [permissions documentation](https://fleetdm.com/docs/using-fleet/permissions) + +# Smoke Tests +Smoke tests are limited to core functionality and serve as a sanity test. If smoke tests are failing, a release cannot proceed. + +## Prerequisites + +1. `fleetctrl preview` is set up and running the desired test version using `--tag` parameters. +2. Unless you are explicitly testing older browser versions, browser is up to date. +3. Certificate & flagfile are in place to create new host. + +## Instructions + + + + + + + + +
    Test nameStep instructionsExpected resultpass/fail
    $Name{what a tester should do}{what a tester should see when they do that}pass/fail
    Update flow + +1. Prereq: previous version is set up with data. +2. Update fleet to testing version +No data should be lost in updatepass/fail
    Login flow + +1. navigate to the login page and attempt to login with both valid and invalid credentials to verify some combination of expected results. +2. Login with SSO + + +1. text fields prompt when blank +2. correct error message is "authentication failed" +3. forget password link prompts for email +4. valid credentials result in a successful login. pass/fail
    Query flowCreate, edit, run, and delete queries. + +1. permissions regarding creating/editing/deleting queries are up to date with documentation +2. syntax errors result in error messaging +3. queries can be run manually +pass/fail
    Host FlowVerify a new host can be added and removed following modal instructions using your own device. + +1. Host is added via command line +2. Host serial number and date added are accurate +3. Host is not visible after it is deleted +4. Warning and informational modals show when expected and make sense +pass/fail
    + +# Notes + +* {any notes} \ No newline at end of file From 2c6fbe4ea240ef2da183bda1b18a938935c449c6 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Mon, 27 Sep 2021 13:12:08 -0400 Subject: [PATCH 78/82] update deployment faq (#2216) * update deployment faq * Update docs/02-Deploying/FAQ.md Co-authored-by: Zach Wasserman Co-authored-by: Zach Wasserman --- docs/02-Deploying/FAQ.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/02-Deploying/FAQ.md b/docs/02-Deploying/FAQ.md index 84f8628337..5ed8391279 100644 --- a/docs/02-Deploying/FAQ.md +++ b/docs/02-Deploying/FAQ.md @@ -13,6 +13,7 @@ - [Is Fleet available as a SaaS product?](#is-fleet-available-as-a-saas-product) - [Is Fleet compatible with X flavor of MySQL?](#is-fleet-compatible-with-x-flavor-of-mysql) - [What are the MySQL user access requirements?](#what-are-the-mysql-user-requirements) +- [What is duplicate enrollment and how do I fix it?](#what-is-duplicate-enrollment-and-how-do-i-fix-it) ## How do I get support for working with Fleet? @@ -108,4 +109,10 @@ Fleet is built to run on MySQL 5.7 or above. However, particularly with AWS Auro ## What are the MySQL user requirements? -The user `fleet prepare db` (via environment variable `FLEET_MYSQL_USERNAME` or command line flag `--mysql_username=`) uses to interact with the database needs to be able to create, alter, and drop tables as well as the ability to create temporary tables. \ No newline at end of file +The user `fleet prepare db` (via environment variable `FLEET_MYSQL_USERNAME` or command line flag `--mysql_username=`) uses to interact with the database needs to be able to create, alter, and drop tables as well as the ability to create temporary tables. + +## What is duplicate enrollment and how do I fix it? + +Duplicate host enrollment is when more than one host enrolls in Fleet using the same identifier (hardware UUID or osquery generated UUID). This can be caused by cloning a VM Image with an already enrolled +osquery client. To resolve the issues, it's advised to configure `--osquery_host_identifier` to `uuid`, and then delete the single host record for that whole set of hosts in the Fleet UI. You can find more information about +[host identifiers here](https://github.com/fleetdm/fleet/blob/main/docs/02-Deploying/02-Configuration.md#osquery_host_identifier). \ No newline at end of file From 2033d8208c261a4f727d83d8a08d844847270a63 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 27 Sep 2021 16:27:38 -0300 Subject: [PATCH 79/82] Add policy updated at (#2246) * wip * Add policy updated at interval and update the UI to use that * Update rest api * Fix tests --- changes/add-policy-updated-at | 1 + .../expectedHostDetailResponseJson.json | 2 +- .../expectedHostDetailResponseYaml.yml | 1 + .../testdata/expectedListHostsJson.json | 4 +- .../testdata/expectedListHostsYaml.yml | 2 + docs/01-Using-Fleet/03-REST-API.md | 3 +- docs/02-Deploying/02-Configuration.md | 17 ++++++ .../ManagePoliciesPage/ManagePoliciesPage.tsx | 4 +- server/config/config.go | 5 ++ server/datastore/mysql/delete_test.go | 1 + server/datastore/mysql/hosts.go | 12 +++- server/datastore/mysql/hosts_test.go | 35 ++++++++++++ server/datastore/mysql/labels_test.go | 12 ++++ ...20210927143115_AddPolicyUpdatedAtColumn.go | 29 ++++++++++ server/datastore/mysql/migrations_test.go | 57 ------------------- server/datastore/mysql/mysql_test.go | 1 + server/datastore/mysql/packs_test.go | 2 + server/datastore/mysql/policies_test.go | 4 ++ server/datastore/mysql/schema.sql | 5 +- server/datastore/mysql/statistics_test.go | 2 + server/datastore/mysql/targets_test.go | 3 + server/datastore/mysql/unicode_test.go | 1 + server/fleet/app.go | 1 + server/fleet/hosts.go | 1 + server/service/integration_core_test.go | 2 + server/service/integration_logger_test.go | 3 + server/service/service_appconfig.go | 5 +- server/service/service_hosts_test.go | 1 + server/service/service_osquery.go | 16 ++++-- server/service/service_osquery_test.go | 39 ++++++++++++- server/test/new_objects.go | 1 + 31 files changed, 196 insertions(+), 76 deletions(-) create mode 100644 changes/add-policy-updated-at create mode 100644 server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go diff --git a/changes/add-policy-updated-at b/changes/add-policy-updated-at new file mode 100644 index 0000000000..5a3d257a7e --- /dev/null +++ b/changes/add-policy-updated-at @@ -0,0 +1 @@ +* Add interval to policies to prevent running them every time the host checks in. diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json index 8f4906c363..aed67bf5e4 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json @@ -1 +1 @@ -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"labels":[],"packs":[],"status":"mia","display_text":"test_host"}} \ No newline at end of file +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","policy_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"labels":[],"packs":[],"status":"mia","display_text":"test_host"}} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml index caf6b03972..2642bf768c 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml @@ -34,6 +34,7 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" + policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/cmd/fleetctl/testdata/expectedListHostsJson.json b/cmd/fleetctl/testdata/expectedListHostsJson.json index 9d90c3914f..1945360d46 100644 --- a/cmd/fleetctl/testdata/expectedListHostsJson.json +++ b/cmd/fleetctl/testdata/expectedListHostsJson.json @@ -1,2 +1,2 @@ -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host"}} -{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host2","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host2","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host2"}} \ No newline at end of file +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","policy_updated_at":"0001-01-01T00:00:00Z","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host"}} +{"kind":"host","apiVersion":"v1","spec":{"created_at":"0001-01-01T00:00:00Z","updated_at":"0001-01-01T00:00:00Z","id":0,"detail_updated_at":"0001-01-01T00:00:00Z","label_updated_at":"0001-01-01T00:00:00Z","last_enrolled_at":"0001-01-01T00:00:00Z","seen_time":"0001-01-01T00:00:00Z","refetch_requested":false,"hostname":"test_host2","uuid":"","platform":"","osquery_version":"","os_version":"","build":"","platform_like":"","policy_updated_at":"0001-01-01T00:00:00Z","code_name":"","uptime":0,"memory":0,"cpu_type":"","cpu_subtype":"","cpu_brand":"","cpu_physical_cores":0,"cpu_logical_cores":0,"hardware_vendor":"","hardware_model":"","hardware_version":"","hardware_serial":"","computer_name":"test_host2","primary_ip":"","primary_mac":"","distributed_interval":0,"config_tls_refresh":0,"logger_tls_period":0,"team_id":null,"pack_stats":null,"team_name":null,"gigs_disk_space_available":0,"percent_disk_space_available":0,"status":"mia","display_text":"test_host2"}} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedListHostsYaml.yml b/cmd/fleetctl/testdata/expectedListHostsYaml.yml index 2b75527a0f..b667063c25 100644 --- a/cmd/fleetctl/testdata/expectedListHostsYaml.yml +++ b/cmd/fleetctl/testdata/expectedListHostsYaml.yml @@ -32,6 +32,7 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" + policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false @@ -76,6 +77,7 @@ spec: percent_disk_space_available: 0 platform: "" platform_like: "" + policy_updated_at: "0001-01-01T00:00:00Z" primary_ip: "" primary_mac: "" refetch_requested: false diff --git a/docs/01-Using-Fleet/03-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md index 9d0629e1d8..e2adf152c7 100644 --- a/docs/01-Using-Fleet/03-REST-API.md +++ b/docs/01-Using-Fleet/03-REST-API.md @@ -5044,7 +5044,8 @@ None. } }, "update_interval": { - "osquery_detail": 3600000000000 + "osquery_detail": 3600000000000, + "osquery_policy": 3600000000000 }, } ``` diff --git a/docs/02-Deploying/02-Configuration.md b/docs/02-Deploying/02-Configuration.md index c959381768..feb46c52a4 100644 --- a/docs/02-Deploying/02-Configuration.md +++ b/docs/02-Deploying/02-Configuration.md @@ -681,6 +681,23 @@ Valid time units are `s`, `m`, `h`. osquery: label_update_interval: 30m ``` + +###### osquery_policy_update_interval + +The interval at which Fleet will ask osquery agents to update their results for policy queries. + +Setting this to a higher value can reduce baseline load on the Fleet server in larger deployments. + +Valid time units are `s`, `m`, `h`. + +- Default value: `1h` +- Environment variable: `FLEET_OSQUERY_POLICY_UPDATE_INTERVAL` +- Config file format: + + ``` + osquery: + policy_update_interval: 30m + ``` ###### osquery_detail_update_interval diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index b3fae239e1..c55ed15ff2 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -25,7 +25,7 @@ import RemovePoliciesModal from "./components/RemovePoliciesModal"; const baseClass = "manage-policies-page"; const DOCS_LINK = - "https://fleetdm.com/docs/deploying/configuration#osquery_detail_update_interval"; + "https://fleetdm.com/docs/deploying/configuration#osquery_policy_update_interval"; interface IRootState { app: { config: IConfig; @@ -93,7 +93,7 @@ const ManagePolicyPage = (): JSX.Element => { try { const response = await configAPI.loadAll(); const interval = secondsToHms( - inMilliseconds(response.update_interval.osquery_detail) / 1000 + inMilliseconds(response.update_interval.osquery_policy) / 1000 ); setUpdateInterval(interval); } catch (error) { diff --git a/server/config/config.go b/server/config/config.go index 5c666118a3..d81baa58ba 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -91,6 +91,7 @@ type OsqueryConfig struct { StatusLogPlugin string `yaml:"status_log_plugin"` ResultLogPlugin string `yaml:"result_log_plugin"` LabelUpdateInterval time.Duration `yaml:"label_update_interval"` + PolicyUpdateInterval time.Duration `yaml:"policy_update_interval"` DetailUpdateInterval time.Duration `yaml:"detail_update_interval"` StatusLogFile string `yaml:"status_log_file"` ResultLogFile string `yaml:"result_log_file"` @@ -299,6 +300,8 @@ func (man Manager) addConfigs() { "Log plugin to use for result logs") man.addConfigDuration("osquery.label_update_interval", 1*time.Hour, "Interval to update host label membership (i.e. 1h)") + man.addConfigDuration("osquery.policy_update_interval", 1*time.Hour, + "Interval to update host policy membership (i.e. 1h)") man.addConfigDuration("osquery.detail_update_interval", 1*time.Hour, "Interval to update host details (i.e. 1h)") man.addConfigString("osquery.status_log_file", "", @@ -464,6 +467,7 @@ func (man Manager) LoadConfig() FleetConfig { StatusLogFile: man.getConfigString("osquery.status_log_file"), ResultLogFile: man.getConfigString("osquery.result_log_file"), LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"), + PolicyUpdateInterval: man.getConfigDuration("osquery.policy_update_interval"), DetailUpdateInterval: man.getConfigDuration("osquery.detail_update_interval"), EnableLogRotation: man.getConfigBool("osquery.enable_log_rotation"), MaxJitterPercent: man.getConfigInt("osquery.max_jitter_percent"), @@ -752,6 +756,7 @@ func TestConfig() FleetConfig { StatusLogPlugin: "filesystem", ResultLogPlugin: "filesystem", LabelUpdateInterval: 1 * time.Hour, + PolicyUpdateInterval: 1 * time.Hour, DetailUpdateInterval: 1 * time.Hour, MaxJitterPercent: 0, }, diff --git a/server/datastore/mysql/delete_test.go b/server/datastore/mysql/delete_test.go index e193b2b8c5..ea039461d2 100644 --- a/server/datastore/mysql/delete_test.go +++ b/server/datastore/mysql/delete_test.go @@ -35,6 +35,7 @@ func testDeleteEntity(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: t.Name() + "1", UUID: t.Name() + "1", diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 2af56b0e2d..6e558b8091 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -21,6 +21,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, osquery_host_id, detail_updated_at, label_updated_at, + policy_updated_at, node_key, hostname, uuid, @@ -32,7 +33,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, seen_time, team_id ) - VALUES( ?,?,?,?,?,?,?,?,?,?,?,?,? ) + VALUES( ?,?,?,?,?,?,?,?,?,?,?,?,?,? ) ` result, err := d.writer.ExecContext( ctx, @@ -40,6 +41,7 @@ func (d *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host, host.OsqueryHostID, host.DetailUpdatedAt, host.LabelUpdatedAt, + host.PolicyUpdatedAt, host.NodeKey, host.Hostname, host.UUID, @@ -65,6 +67,7 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { UPDATE hosts SET detail_updated_at = ?, label_updated_at = ?, + policy_updated_at = ?, node_key = ?, hostname = ?, uuid = ?, @@ -101,6 +104,7 @@ func (d *Datastore) SaveHost(ctx context.Context, host *fleet.Host) error { _, err := tx.ExecContext(ctx, sqlStatement, host.DetailUpdatedAt, host.LabelUpdatedAt, + host.PolicyUpdatedAt, host.NodeKey, host.Hostname, host.UUID, @@ -492,13 +496,14 @@ func (d *Datastore) EnrollHost(ctx context.Context, osqueryHostID, nodeKey strin INSERT INTO hosts ( detail_updated_at, label_updated_at, + policy_updated_at, osquery_host_id, seen_time, node_key, team_id - ) VALUES (?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, teamID) + result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, teamID) if err != nil { return errors.Wrap(err, "insert host") @@ -561,6 +566,7 @@ func (d *Datastore) AuthenticateHost(ctx context.Context, nodeKey string) (*flee updated_at, detail_updated_at, label_updated_at, + policy_updated_at, node_key, hostname, uuid, diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 222ca90b06..37cc857e5f 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -92,9 +92,11 @@ func TestHosts(t *testing.T) { } func testHostsSave(t *testing.T, ds *Datastore) { + policyUpdatedAt := time.Now().UTC().Truncate(time.Second) host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: policyUpdatedAt, SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -114,6 +116,7 @@ func testHostsSave(t *testing.T, ds *Datastore) { assert.Equal(t, "bar.local", host.Hostname) assert.Equal(t, "192.168.1.1", host.PrimaryIP) assert.Equal(t, "30-65-EC-6F-C4-58", host.PrimaryMac) + assert.Equal(t, policyUpdatedAt.UTC(), host.PolicyUpdatedAt) additionalJSON := json.RawMessage(`{"foobar": "bim"}`) host.Additional = &additionalJSON @@ -146,6 +149,7 @@ func testHostsDeleteWithSoftware(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -179,6 +183,7 @@ func testHostsSavePackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -291,6 +296,7 @@ func testHostsSavePackStatsOverwrites(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -434,6 +440,7 @@ func testHostsIgnoresTeamPackStats(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -513,6 +520,7 @@ func testHostsDelete(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -532,6 +540,7 @@ func testHostsListFilterAdditional(t *testing.T, ds *Datastore) { h, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "foobar", NodeKey: "nodekey", @@ -570,6 +579,7 @@ func testHostsListStatus(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -607,6 +617,7 @@ func testHostsListQuery(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -759,6 +770,7 @@ func testHostsSearch(t *testing.T, ds *Datastore) { OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -770,6 +782,7 @@ func testHostsSearch(t *testing.T, ds *Datastore) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -781,6 +794,7 @@ func testHostsSearch(t *testing.T, ds *Datastore) { OsqueryHostID: "99999", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "3", UUID: "abc-def-ghi", @@ -850,6 +864,7 @@ func testHostsSearchLimit(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -882,6 +897,7 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: mockClock.Now().Add(-30 * time.Second), LabelUpdatedAt: mockClock.Now().Add(-30 * time.Second), + PolicyUpdatedAt: mockClock.Now().Add(-30 * time.Second), SeenTime: mockClock.Now().Add(-30 * time.Second), }) require.Nil(t, err) @@ -896,6 +912,7 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "2", DetailUpdatedAt: mockClock.Now().Add(-1 * time.Minute), LabelUpdatedAt: mockClock.Now().Add(-1 * time.Minute), + PolicyUpdatedAt: mockClock.Now().Add(-1 * time.Minute), SeenTime: mockClock.Now().Add(-1 * time.Minute), }) require.Nil(t, err) @@ -910,6 +927,7 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "3", DetailUpdatedAt: mockClock.Now().Add(-1 * time.Hour), LabelUpdatedAt: mockClock.Now().Add(-1 * time.Hour), + PolicyUpdatedAt: mockClock.Now().Add(-1 * time.Hour), SeenTime: mockClock.Now().Add(-1 * time.Hour), }) require.Nil(t, err) @@ -924,6 +942,7 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { NodeKey: "4", DetailUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), LabelUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), + PolicyUpdatedAt: mockClock.Now().Add(-35 * (24 * time.Hour)), SeenTime: mockClock.Now().Add(-35 * (24 * time.Hour)), }) require.Nil(t, err) @@ -956,6 +975,7 @@ func testHostsMarkSeen(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, + PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) assert.Nil(t, err) @@ -992,6 +1012,7 @@ func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, + PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) require.Nil(t, err) @@ -1003,6 +1024,7 @@ func testHostsMarkSeenMany(t *testing.T, ds *Datastore) { NodeKey: "2", DetailUpdatedAt: aDayAgo, LabelUpdatedAt: aDayAgo, + PolicyUpdatedAt: aDayAgo, SeenTime: aDayAgo, }) require.Nil(t, err) @@ -1048,6 +1070,7 @@ func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { NodeKey: "1", DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), }) require.Nil(t, err) @@ -1061,6 +1084,7 @@ func testHostsCleanupIncoming(t *testing.T, ds *Datastore) { OsqueryVersion: "3.2.3", DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), }) require.Nil(t, err) @@ -1089,6 +1113,7 @@ func testHostsIDsByName(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("host%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -1109,6 +1134,7 @@ func testHostsAdditional(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "foobar", NodeKey: "nodekey", @@ -1181,6 +1207,7 @@ func testHostsByIdentifier(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: fmt.Sprintf("osquery_host_id_%d", i), NodeKey: fmt.Sprintf("node_key_%d", i), @@ -1265,6 +1292,7 @@ func testHostsSaveUsers(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1334,6 +1362,7 @@ func testHostsSaveUsersWithoutUid(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1389,6 +1418,7 @@ func addHostSeenLast(t *testing.T, ds fleet.Datastore, i, days int) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-1 * time.Duration(days) * 24 * time.Hour), OsqueryHostID: fmt.Sprintf("%d", i), NodeKey: fmt.Sprintf("%d", i), @@ -1423,6 +1453,7 @@ func testHostsListByPolicy(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), @@ -1477,6 +1508,7 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1491,6 +1523,7 @@ func testHostsSaveTonsOfUsers(t *testing.T, ds *Datastore) { host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -1640,6 +1673,7 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { host1, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -1654,6 +1688,7 @@ func testHostsSavePackStatsConcurrent(t *testing.T, ds *Datastore) { host2, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index 3aa2ee7f8b..c610ce7859 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -240,6 +240,7 @@ func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -251,6 +252,7 @@ func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "2", NodeKey: "2", @@ -262,6 +264,7 @@ func testLabelsListHostsInLabel(t *testing.T, db *Datastore) { h3, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "3", NodeKey: "3", @@ -302,6 +305,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -314,6 +318,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: lastSeenTime, LabelUpdatedAt: lastSeenTime, + PolicyUpdatedAt: lastSeenTime, SeenTime: lastSeenTime, OsqueryHostID: "2", NodeKey: "2", @@ -355,6 +360,7 @@ func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -367,6 +373,7 @@ func testLabelsListHostsInLabelAndTeamFilter(t *testing.T, db *Datastore) { h2, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: lastSeenTime, LabelUpdatedAt: lastSeenTime, + PolicyUpdatedAt: lastSeenTime, SeenTime: lastSeenTime, OsqueryHostID: "2", NodeKey: "2", @@ -445,6 +452,7 @@ func testLabelsListUniqueHostsInLabels(t *testing.T, db *Datastore) { h, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), @@ -515,6 +523,7 @@ func setupLabelSpecsTest(t *testing.T, ds fleet.Datastore) []*fleet.LabelSpec { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: strconv.Itoa(i), NodeKey: strconv.Itoa(i), @@ -597,6 +606,7 @@ func testLabelsSave(t *testing.T, db *Datastore) { h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -658,6 +668,7 @@ func testLabelsRecordNonexistentQueryLabelExecution(t *testing.T, db *Datastore) h1, err := db.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), OsqueryHostID: "1", NodeKey: "1", @@ -682,6 +693,7 @@ func testLabelMembershipCleanup(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go b/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go new file mode 100644 index 0000000000..43385178ee --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210927143115_AddPolicyUpdatedAtColumn.go @@ -0,0 +1,29 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210927143115, Down_20210927143115) +} + +func Up_20210927143115(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE hosts ADD COLUMN policy_updated_at timestamp NOT NULL DEFAULT '2000-01-01 00:00:00'") + if err != nil { + return errors.Wrap(err, "adding policy_updated_at column") + } + + _, err = tx.Exec("delete from policy_membership_history") + if err != nil { + return errors.Wrap(err, "clearing policy_membership_history") + } + + return err +} + +func Down_20210927143115(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations_test.go b/server/datastore/mysql/migrations_test.go index 2398c5d8ac..57d469eec4 100644 --- a/server/datastore/mysql/migrations_test.go +++ b/server/datastore/mysql/migrations_test.go @@ -5,12 +5,9 @@ import ( "context" "os/exec" "testing" - "time" "github.com/fleetdm/fleet/v4/server/config" - "github.com/fleetdm/fleet/v4/server/datastore/mysql/migrations/tables" "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -68,57 +65,3 @@ func createMySQLDSForMigrationTests(t *testing.T, dbName string) *Datastore { require.NoError(t, err) return ds } - -func Test20210819131107_AddCascadeToHostSoftware(t *testing.T) { - ds := createMySQLDSForMigrationTests(t, t.Name()) - defer ds.Close() - - for { - version, err := tables.MigrationClient.GetDBVersion(ds.writer.DB) - require.NoError(t, err) - - // break right before the the constraint migration - if version == 20210818182258 { - break - } - require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) - } - - host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) - host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) - - soft1 := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, - {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, - }, - } - host1.HostSoftware = soft1 - soft2 := fleet.HostSoftware{ - Modified: true, - Software: []fleet.Software{ - {Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}, - {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, - {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, - }, - } - host2.HostSoftware = soft2 - host2.Modified = true - - require.NoError(t, ds.SaveHostSoftware(context.Background(), host1)) - require.NoError(t, ds.SaveHostSoftware(context.Background(), host2)) - - require.NoError(t, ds.DeleteHost(context.Background(), host1.ID)) - - t.Log("Done adding software...") - startTime := time.Now() - require.NoError(t, tables.MigrationClient.UpByOne(ds.writer.DB, "")) - t.Log("took", time.Since(startTime)) - - // Make sure we don't delete more than we need - hostCheck, err := ds.Host(context.Background(), host2.ID) - require.NoError(t, err) - require.NoError(t, ds.LoadHostSoftware(context.Background(), hostCheck)) - require.Len(t, hostCheck.Software, 3) -} diff --git a/server/datastore/mysql/mysql_test.go b/server/datastore/mysql/mysql_test.go index 68b40cbcdd..3e10b99d20 100644 --- a/server/datastore/mysql/mysql_test.go +++ b/server/datastore/mysql/mysql_test.go @@ -50,6 +50,7 @@ func TestDatastoreReplica(t *testing.T) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 43d97c94d8..c571b3b36f 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -599,6 +599,7 @@ func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -644,6 +645,7 @@ func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { host, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 2ea27f71e0..8897d98995 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -78,6 +78,7 @@ func testPoliciesMembershipView(t *testing.T, ds *Datastore) { OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -89,6 +90,7 @@ func testPoliciesMembershipView(t *testing.T, ds *Datastore) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", @@ -234,6 +236,7 @@ func TestPolicyQueriesForHost(t *testing.T) { OsqueryHostID: "1234", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -249,6 +252,7 @@ func TestPolicyQueriesForHost(t *testing.T) { OsqueryHostID: "5679", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index d93fef6439..00f3bea508 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -221,6 +221,7 @@ CREATE TABLE `hosts` ( `team_id` int(10) unsigned DEFAULT NULL, `gigs_disk_space_available` float NOT NULL DEFAULT '0', `percent_disk_space_available` float NOT NULL DEFAULT '0', + `policy_updated_at` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', PRIMARY KEY (`id`), UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`), UNIQUE KEY `idx_host_unique_nodekey` (`node_key`), @@ -308,9 +309,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=104 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=105 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( diff --git a/server/datastore/mysql/statistics_test.go b/server/datastore/mysql/statistics_test.go index 420effa5b9..34583b0ab7 100644 --- a/server/datastore/mysql/statistics_test.go +++ b/server/datastore/mysql/statistics_test.go @@ -31,6 +31,7 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { _, err := ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1", UUID: "1", @@ -62,6 +63,7 @@ func testStatisticsShouldSend(t *testing.T, ds *Datastore) { _, err = ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "2", UUID: "2", diff --git a/server/datastore/mysql/targets_test.go b/server/datastore/mysql/targets_test.go index b3b8957e37..1f2cd2000c 100644 --- a/server/datastore/mysql/targets_test.go +++ b/server/datastore/mysql/targets_test.go @@ -47,6 +47,7 @@ func testTargetsCountHosts(t *testing.T, ds *Datastore) { OsqueryHostID: strconv.Itoa(hostCount), DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), NodeKey: strconv.Itoa(hostCount), DistributedInterval: distributedInterval, @@ -247,6 +248,7 @@ func testTargetsHostIDsInTargets(t *testing.T, ds *Datastore) { NodeKey: strconv.Itoa(hostCount), DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), }) require.Nil(t, err) @@ -325,6 +327,7 @@ func testTargetsHostIDsInTargetsTeam(t *testing.T, ds *Datastore) { OsqueryHostID: strconv.Itoa(hostCount), DetailUpdatedAt: mockClock.Now(), LabelUpdatedAt: mockClock.Now(), + PolicyUpdatedAt: mockClock.Now(), SeenTime: mockClock.Now(), NodeKey: strconv.Itoa(hostCount), DistributedInterval: distributedInterval, diff --git a/server/datastore/mysql/unicode_test.go b/server/datastore/mysql/unicode_test.go index 64bfb3adb7..a06a991bfa 100644 --- a/server/datastore/mysql/unicode_test.go +++ b/server/datastore/mysql/unicode_test.go @@ -33,6 +33,7 @@ func TestUnicode(t *testing.T) { Hostname: "🍌", DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), }) require.Nil(t, err) diff --git a/server/fleet/app.go b/server/fleet/app.go index 3b141315f6..93f7644607 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -322,6 +322,7 @@ type Logging struct { type UpdateIntervalConfig struct { OSQueryDetail time.Duration `json:"osquery_detail"` + OSQueryPolicy time.Duration `json:"osquery_policy"` } type LoggingPlugin struct { diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 5c0e19f1d9..5ef79d568f 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -64,6 +64,7 @@ type Host struct { OsqueryHostID string `json:"-" db:"osquery_host_id"` DetailUpdatedAt time.Time `json:"detail_updated_at" db:"detail_updated_at"` // Time that the host details were last updated LabelUpdatedAt time.Time `json:"label_updated_at" db:"label_updated_at"` // Time that the host labels were last updated + PolicyUpdatedAt time.Time `json:"policy_updated_at" db:"policy_updated_at"` // Time that the host policies were last updated LastEnrolledAt time.Time `json:"last_enrolled_at" db:"last_enrolled_at"` // Time that the host last enrolled SeenTime time.Time `json:"seen_time" db:"seen_time"` // Time that the host was last "seen" RefetchRequested bool `json:"refetch_requested" db:"refetch_requested"` diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 3922225b70..25dafc96ca 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -266,6 +266,7 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { host, err := s.ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: t.Name() + "1", UUID: t.Name() + "1", @@ -327,6 +328,7 @@ func (s *integrationTestSuite) TestGlobalPolicies() { _, err := s.ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), diff --git a/server/service/integration_logger_test.go b/server/service/integration_logger_test.go index 0816e6e8e8..2f39f97496 100644 --- a/server/service/integration_logger_test.go +++ b/server/service/integration_logger_test.go @@ -102,6 +102,7 @@ func (s *integrationLoggerTestSuite) TestOsqueryEndpointsLogErrors() { _, err := s.ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: t.Name() + "1234", UUID: "1", @@ -129,6 +130,7 @@ func (s *integrationLoggerTestSuite) TestSubmitStatusLog() { _, err := s.ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: t.Name() + "1234", UUID: "1", @@ -157,6 +159,7 @@ func (s *integrationLoggerTestSuite) TestEnrollAgentLogsErrors() { _, err := s.ds.NewHost(context.Background(), &fleet.Host{ DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), SeenTime: time.Now(), NodeKey: "1234", UUID: "1", diff --git a/server/service/service_appconfig.go b/server/service/service_appconfig.go index e15ecf851d..1c9ce979e7 100644 --- a/server/service/service_appconfig.go +++ b/server/service/service_appconfig.go @@ -181,7 +181,10 @@ func (svc *Service) SetupRequired(ctx context.Context) (bool, error) { } func (svc *Service) UpdateIntervalConfig(ctx context.Context) (*fleet.UpdateIntervalConfig, error) { - return &fleet.UpdateIntervalConfig{OSQueryDetail: svc.config.Osquery.DetailUpdateInterval}, nil + return &fleet.UpdateIntervalConfig{ + OSQueryDetail: svc.config.Osquery.DetailUpdateInterval, + OSQueryPolicy: svc.config.Osquery.PolicyUpdateInterval, + }, nil } func (svc *Service) LoggingConfig(ctx context.Context) (*fleet.Logging, error) { diff --git a/server/service/service_hosts_test.go b/server/service/service_hosts_test.go index 7f276443b7..d288d2ff6e 100644 --- a/server/service/service_hosts_test.go +++ b/server/service/service_hosts_test.go @@ -32,6 +32,7 @@ func TestListHosts(t *testing.T) { SeenTime: storedTime, DetailUpdatedAt: time.Now(), LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), }) require.NoError(t, err) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index dce97d9332..6787aa7c7b 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -490,13 +490,15 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin queries[hostDistributedQueryPrefix+name] = query } - policyQueries, err := svc.ds.PolicyQueriesForHost(ctx, &host) - if err != nil { - return nil, 0, osqueryError{message: "retrieving policy queries: " + err.Error()} - } + if svc.shouldUpdate(host.PolicyUpdatedAt, svc.config.Osquery.PolicyUpdateInterval) { + policyQueries, err := svc.ds.PolicyQueriesForHost(ctx, &host) + if err != nil { + return nil, 0, osqueryError{message: "retrieving policy queries: " + err.Error()} + } - for name, query := range policyQueries { - queries[hostPolicyQueryPrefix+name] = query + for name, query := range policyQueries { + queries[hostPolicyQueryPrefix+name] = query + } } accelerate := uint(0) @@ -707,6 +709,8 @@ func (svc *Service) SubmitDistributedQueryResults( } if len(policyResults) > 0 { + host.Modified = true + host.PolicyUpdatedAt = svc.clock.Now() err = svc.ds.RecordPolicyQueryExecutions(ctx, &host, policyResults, svc.clock.Now()) if err != nil { logging.WithErr(ctx, err) diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index e2c6eb9ce0..2f0d3f0fe9 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -1819,7 +1819,8 @@ func TestPolicyQueries(t *testing.T) { ds.HostFunc = func(ctx context.Context, id uint) (*fleet.Host, error) { return host, nil } - ds.SaveHostFunc = func(ctx context.Context, host *fleet.Host) error { + ds.SaveHostFunc = func(ctx context.Context, gotHost *fleet.Host) error { + host = gotHost return nil } ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { @@ -1872,4 +1873,40 @@ func TestPolicyQueries(t *testing.T) { require.NotNil(t, recordedResults[1]) require.Equal(t, true, *recordedResults[1]) require.Nil(t, recordedResults[2]) + + ctx = hostctx.NewContext(context.Background(), *host) + queries, _, err = svc.GetDistributedQueries(ctx) + require.NoError(t, err) + require.Len(t, queries, expectedDetailQueries) + + // After the first time we get policies and update the host, then there shouldn't be any policies + hasAnyPolicy := false + for name := range queries { + if strings.HasPrefix(name, hostPolicyQueryPrefix) { + hasAnyPolicy = true + break + } + } + assert.False(t, hasAnyPolicy) + + // Let's move time forward, there should be policies now + mockClock.AddTime(2 * time.Hour) + + queries, _, err = svc.GetDistributedQueries(ctx) + require.NoError(t, err) + require.Len(t, queries, expectedDetailQueries+2) + + hasPolicy1, hasPolicy2 = false, false + for name := range queries { + if strings.HasPrefix(name, hostPolicyQueryPrefix) { + if name[len(hostPolicyQueryPrefix):] == "1" { + hasPolicy1 = true + } + if name[len(hostPolicyQueryPrefix):] == "2" { + hasPolicy2 = true + } + } + } + assert.True(t, hasPolicy1) + assert.True(t, hasPolicy2) } diff --git a/server/test/new_objects.go b/server/test/new_objects.go index aaa09a7f11..171239ad79 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -105,6 +105,7 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t UUID: uuid, DetailUpdatedAt: now, LabelUpdatedAt: now, + PolicyUpdatedAt: now, SeenTime: now, OsqueryHostID: osqueryHostID, }) From 968854c721969c06e92a830fbb540751fd81e121 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 27 Sep 2021 16:28:02 -0300 Subject: [PATCH 80/82] Correct documentation typo for interval (#2248) --- docs/02-Deploying/02-Configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/02-Deploying/02-Configuration.md b/docs/02-Deploying/02-Configuration.md index feb46c52a4..91ab7e63d1 100644 --- a/docs/02-Deploying/02-Configuration.md +++ b/docs/02-Deploying/02-Configuration.md @@ -1365,13 +1365,13 @@ When `current_instance_checks` is set to `auto` (the default), Fleet instances w How often vulnerabilities are checked. -- Default value: `1hr` +- Default value: `1h` - Environment variable: `FLEET_VULNERABILITIES_PERIODICITY` - Config file format: ``` vulnerabilities: - periodicity: 1hr + periodicity: 1h ``` ###### cpe_database_url From c91d9c2fc3ce6251a7c49bab27a4bcae7eef7fed Mon Sep 17 00:00:00 2001 From: Mike McNeil Date: Mon, 27 Sep 2021 16:01:49 -0500 Subject: [PATCH 81/82] reduce Slack notifications by limiting activity from maintainers (#2232) * reduce Slack notifications by limiting to activity from maintainers * rename var --- .../webhooks/receive-from-github.js | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/website/api/controllers/webhooks/receive-from-github.js b/website/api/controllers/webhooks/receive-from-github.js index b6dac23eee..20e5abd03e 100644 --- a/website/api/controllers/webhooks/receive-from-github.js +++ b/website/api/controllers/webhooks/receive-from-github.js @@ -25,8 +25,24 @@ module.exports = { let GitHub = require('machinepack-github'); let GREEN_LABEL_COLOR = 'C2E0C6';// « Used in multiple places below. - let GITHUB_USERNAMES_OF_BOTS = [// « Used in multiple places below. - 'sailsbot' + let GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS = [// « Used in multiple places below. + 'sailsbot', + 'fleet-release', + 'noahtalerman', + 'mike-j-thomas', + 'mikermcneil', + 'lukeheath', + 'zwass', + 'rlynnj11', + 'martavis', + 'rachelelysia', + 'gillespi314', + 'chiiph', + 'mna', + 'edwardsb', + 'alphabrevity', + 'eashaw', + 'drewbakerfdm' ]; let GITHUB_USERNAME_OF_DRI_FOR_LABELS = 'noahtalerman';// « Used below @@ -94,7 +110,7 @@ module.exports = { // `For help with questions about Sails, [click here](http://sailsjs.com/support).\n`; // } // } else { - // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); + // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); // if (wasReopenedByBot) { // newBotComment = '';// « checked below // } else { @@ -146,7 +162,7 @@ module.exports = { // if (action === 'edited' && pr.state !== 'open') { // // If this is an edit to an already-closed pull request, then do nothing. // } else if (action === 'reopened') { - // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); + // let wasReopenedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); // if (!wasReopenedByBot) { // let newBotComment = // `Oh hey again, @${issueOrPr.user.login}. Now that this pull request is reopened, it's on our radar. Please let us know if there's any new information we should be aware of!\n`+ @@ -209,7 +225,7 @@ module.exports = { let repo = repository.name; let issueNumber = issueOrPr.number; - let wasPostedByBot = GITHUB_USERNAMES_OF_BOTS.includes(sender.login); + let wasPostedByBot = GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(sender.login); if (!wasPostedByBot) { let greenLabels = _.filter(issueOrPr.labels, ({color}) => color === GREEN_LABEL_COLOR); await sails.helpers.flow.simultaneouslyForEach(greenLabels, async(greenLabel)=>{ @@ -217,7 +233,7 @@ module.exports = { });//∞ß }//fi } else if ( - (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS.includes(comment.user.login))|| + (ghNoun === 'issue_comment' && ['deleted'].includes(action) && !GITHUB_USERNAMES_OF_BOTS_AND_MAINTAINERS.includes(comment.user.login))|| (ghNoun === 'commit_comment' && ['created'].includes(action))|| (ghNoun === 'label' && ['created','edited','deleted'].includes(action) && GITHUB_USERNAME_OF_DRI_FOR_LABELS !== sender.login)||//« exempt label changes made by the directly responsible individual for labels, because otherwise when process changes/fiddlings happen, they can otherwise end up making too much noise in Slack (ghNoun === 'issue_comment' && ['created'].includes(action) && issueOrPr.state !== 'open' && (issueOrPr.closed_at) && ((new Date(issueOrPr.closed_at)).getTime() < Date.now() - 7*24*60*60*1000 ) ) From 63633723d24e39902a42700bf82f7b649c41dd99 Mon Sep 17 00:00:00 2001 From: Mike Thomas <78363703+mike-j-thomas@users.noreply.github.com> Date: Tue, 28 Sep 2021 08:48:03 +0900 Subject: [PATCH 82/82] Update homepage.ejs Updated homepage verbiage. --- website/views/pages/homepage.ejs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/views/pages/homepage.ejs b/website/views/pages/homepage.ejs index d8be8c66ef..18792d2001 100644 --- a/website/views/pages/homepage.ejs +++ b/website/views/pages/homepage.ejs @@ -36,7 +36,7 @@

    - Inventory management + Single source or truth

    Track and segment your enrolled hosts. Search by important details, and zoom in on individual targets. @@ -67,7 +67,7 @@

    - Deployment + Deploy anywhere

    Fleet is self-hosted and self-managed, and can be run within your own data centers or in the cloud.