Allow hosts to check in even if Redis is down (#3506)

This commit is contained in:
Lucas Manuel Rodriguez 2021-12-29 22:06:23 -03:00 committed by GitHub
parent 72d71b1eee
commit aaa5b7ec3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 58 additions and 6 deletions

View file

@ -0,0 +1 @@
* Allow hosts to check in even if Redis is down.

View file

@ -517,12 +517,15 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin
queries[hostLabelQueryPrefix+name] = query
}
liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID)
if err != nil {
return nil, 0, osqueryError{message: "retrieve live queries: " + err.Error()}
}
for name, query := range liveQueries {
queries[hostDistributedQueryPrefix+name] = query
if liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID); err != nil {
// If the live query store fails to fetch queries we still want the hosts
// to receive all the other queries (details, policies, labels, etc.),
// thus we just log the error.
level.Error(svc.logger).Log("op", "QueriesForHost", "err", err)
} else {
for name, query := range liveQueries {
queries[hostDistributedQueryPrefix+name] = query
}
}
policyQueries, err := svc.policyQueriesForHost(ctx, &host)

View file

@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"reflect"
"sort"
"strconv"
@ -2354,3 +2355,50 @@ func TestPolicyWebhooks(t *testing.T) {
}, 5*time.Second, 250*time.Millisecond)
require.NoError(t, err)
}
// If the live query store (Redis) is down we still (see #3503)
// want hosts to get queries and continue to check in.
func TestLiveQueriesFailing(t *testing.T) {
ds := new(mock.Store)
lq := new(live_query.MockLiveQuery)
cfg := config.TestConfig()
buf := new(bytes.Buffer)
logger := log.NewLogfmtLogger(buf)
svc := newTestServiceWithConfig(ds, cfg, nil, lq, TestServerOpts{
Logger: logger,
})
hostID := uint(1)
host := &fleet.Host{
ID: hostID,
Platform: "darwin",
}
lq.On("QueriesForHost", hostID).Return(
map[string]string{},
errors.New("failed to get queries for host"),
)
ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) {
return map[string]string{}, nil
}
ds.HostFunc = func(ctx context.Context, id uint, skipLoadingExtras bool) (*fleet.Host, error) {
return host, nil
}
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil
}
ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) {
return map[string]string{}, nil
}
ctx := hostctx.NewContext(context.Background(), *host)
queries, _, err := svc.GetDistributedQueries(ctx)
require.NoError(t, err)
require.Len(t, queries, expectedDetailQueries)
logs, err := ioutil.ReadAll(buf)
require.NoError(t, err)
require.Contains(t, string(logs), "level=error")
require.Contains(t, string(logs), "failed to get queries for host")
}