From aaa5b7ec3c6a503a7f047153801aef142fcca27c Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez <lucarodriguez@gmail.com>
Date: Wed, 29 Dec 2021 22:06:23 -0300
Subject: [PATCH] Allow hosts to check in even if Redis is down (#3506)

---
 ...issue-3503-if-redis-down-host-can-check-in |  1 +
 server/service/service_osquery.go             | 15 +++---
 server/service/service_osquery_test.go        | 48 +++++++++++++++++++
 3 files changed, 58 insertions(+), 6 deletions(-)
 create mode 100644 changes/issue-3503-if-redis-down-host-can-check-in

diff --git a/changes/issue-3503-if-redis-down-host-can-check-in b/changes/issue-3503-if-redis-down-host-can-check-in
new file mode 100644
index 0000000000..4aa4186cb4
--- /dev/null
+++ b/changes/issue-3503-if-redis-down-host-can-check-in
@@ -0,0 +1 @@
+* Allow hosts to check in even if Redis is down.
diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go
index f58383be7e..1baf98a0b0 100644
--- a/server/service/service_osquery.go
+++ b/server/service/service_osquery.go
@@ -517,12 +517,15 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin
 		queries[hostLabelQueryPrefix+name] = query
 	}
 
-	liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID)
-	if err != nil {
-		return nil, 0, osqueryError{message: "retrieve live queries: " + err.Error()}
-	}
-	for name, query := range liveQueries {
-		queries[hostDistributedQueryPrefix+name] = query
+	if liveQueries, err := svc.liveQueryStore.QueriesForHost(host.ID); err != nil {
+		// If the live query store fails to fetch queries we still want the hosts
+		// to receive all the other queries (details, policies, labels, etc.),
+		// thus we just log the error.
+		level.Error(svc.logger).Log("op", "QueriesForHost", "err", err)
+	} else {
+		for name, query := range liveQueries {
+			queries[hostDistributedQueryPrefix+name] = query
+		}
 	}
 
 	policyQueries, err := svc.policyQueriesForHost(ctx, &host)
diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go
index 6a33572eaa..5489e64ce5 100644
--- a/server/service/service_osquery_test.go
+++ b/server/service/service_osquery_test.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"reflect"
 	"sort"
 	"strconv"
@@ -2354,3 +2355,50 @@ func TestPolicyWebhooks(t *testing.T) {
 	}, 5*time.Second, 250*time.Millisecond)
 	require.NoError(t, err)
 }
+
+// If the live query store (Redis) is down we still (see #3503)
+// want hosts to get queries and continue to check in.
+func TestLiveQueriesFailing(t *testing.T) {
+	ds := new(mock.Store)
+	lq := new(live_query.MockLiveQuery)
+	cfg := config.TestConfig()
+	buf := new(bytes.Buffer)
+	logger := log.NewLogfmtLogger(buf)
+	svc := newTestServiceWithConfig(ds, cfg, nil, lq, TestServerOpts{
+		Logger: logger,
+	})
+
+	hostID := uint(1)
+	host := &fleet.Host{
+		ID:       hostID,
+		Platform: "darwin",
+	}
+	lq.On("QueriesForHost", hostID).Return(
+		map[string]string{},
+		errors.New("failed to get queries for host"),
+	)
+
+	ds.LabelQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) {
+		return map[string]string{}, nil
+	}
+	ds.HostFunc = func(ctx context.Context, id uint, skipLoadingExtras bool) (*fleet.Host, error) {
+		return host, nil
+	}
+	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
+		return &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, nil
+	}
+	ds.PolicyQueriesForHostFunc = func(ctx context.Context, host *fleet.Host) (map[string]string, error) {
+		return map[string]string{}, nil
+	}
+
+	ctx := hostctx.NewContext(context.Background(), *host)
+
+	queries, _, err := svc.GetDistributedQueries(ctx)
+	require.NoError(t, err)
+	require.Len(t, queries, expectedDetailQueries)
+
+	logs, err := ioutil.ReadAll(buf)
+	require.NoError(t, err)
+	require.Contains(t, string(logs), "level=error")
+	require.Contains(t, string(logs), "failed to get queries for host")
+}