Introduce forced failures for the Jira client. (#5088)

cmd/fleet/cron.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/fleetdm/fleet/v4/server/config"
@@ -403,6 +405,24 @@ func cronWorker(
 	}
 	w.Register(jira)
 
+	// create a JiraClient wrapper to introduce forced failures if configured
+	// to do so via the environment variable.
+	var failerClient *worker.TestJiraFailer
+	if forcedFailures := os.Getenv("FLEET_JIRA_CLIENT_FORCED_FAILURES"); forcedFailures != "" {
+		// format is "<modulo number>;<cve1>,<cve2>,<cve3>,..."
+		parts := strings.Split(forcedFailures, ";")
+		if len(parts) == 2 {
+			mod, _ := strconv.Atoi(parts[0])
+			cves := strings.Split(parts[1], ",")
+			if mod > 0 || len(cves) > 0 {
+				failerClient = &worker.TestJiraFailer{
+					FailCallCountModulo: mod,
+					AlwaysFailCVEs:      cves,
+				}
+			}
+		}
+	}
+
 	ticker := time.NewTicker(10 * time.Second)
 	for {
 		select {
@@ -461,7 +481,12 @@ func cronWorker(
 
 		// safe to update the Jira worker as it is not used concurrently
 		jira.FleetURL = appConfig.ServerSettings.ServerURL
-		jira.JiraClient = client
+		if failerClient != nil && strings.Contains(jira.FleetURL, "fleetdm") {
+			failerClient.JiraClient = client
+			jira.JiraClient = failerClient
+		} else {
+			jira.JiraClient = client
+		}
 
 		workCtx, cancel := context.WithTimeout(ctx, lockDuration)
 		if err := w.ProcessJobs(workCtx); err != nil {

server/worker/jira_failer.go

@@ -0,0 +1,53 @@
+package worker
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	jira "github.com/andygrunwald/go-jira"
+)
+
+// TestJiraFailer is an implementation of the JiraClient interface that wraps
+// another JiraClient and introduces forced failures so that error-handling
+// logic can be tested at scale in a real environment (e.g. in the load-testing
+// environment).
+type TestJiraFailer struct {
+	// FailCallCountModulo is the number of calls to execute normally vs
+	// forcing a failure. In other words, it will force a failure every time
+	// callCounts % FailCallCountModulo == 0. If it is <= 0, no forced failure is
+	// introduced based on call counts.
+	FailCallCountModulo int
+
+	// AlwaysFailCVEs is the list of CVEs for which a failure will always be
+	// forced, so that those CVEs never succeed in creating a Jira ticket.
+	AlwaysFailCVEs []string
+
+	// JiraClient is the wrapped Jira client to use for normal calls, when no
+	// forced failure is inserted.
+	JiraClient JiraClient
+
+	callCounts int
+}
+
+// CreateIssue implements the JiraClient and introduces a forced failure if
+// required, otherwise it returns the result of calling
+// f.JiraClient.CreateIssue with the provided arguments.
+func (f *TestJiraFailer) CreateIssue(ctx context.Context, issue *jira.Issue) (*jira.Issue, error) {
+	f.callCounts++
+
+	if issue.Fields != nil && issue.Fields.Summary != "" {
+		s := issue.Fields.Summary
+		for _, cve := range f.AlwaysFailCVEs {
+			if strings.Contains(s, cve) {
+				return nil, fmt.Errorf("always failing CVE %q", cve)
+			}
+		}
+	}
+
+	if f.FailCallCountModulo > 0 && f.callCounts%f.FailCallCountModulo == 0 {
+		return nil, fmt.Errorf("failing due to FailCallCountModulo: callCount=%d", f.callCounts)
+	}
+
+	return f.JiraClient.CreateIssue(ctx, issue)
+}

server/worker/jira_failer_test.go

@@ -0,0 +1,77 @@
+package worker
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/mock"
+	"github.com/fleetdm/fleet/v4/server/service/externalsvc"
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTestJiraFailer(t *testing.T) {
+	ds := new(mock.Store)
+	ds.HostsByCVEFunc = func(ctx context.Context, cve string) ([]*fleet.HostShort, error) {
+		return []*fleet.HostShort{{ID: 1, Hostname: "test"}}, nil
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusCreated)
+		w.Write([]byte(`
+{
+  "id": "10000",
+  "key": "ED-24",
+  "self": "https://your-domain.atlassian.net/rest/api/2/issue/10000",
+  "transition": {
+    "status": 200,
+    "errorCollection": {
+      "errorMessages": [],
+      "errors": {}
+    }
+  }
+}`))
+	}))
+	defer srv.Close()
+
+	// create the real client, that will never fail
+	client, err := externalsvc.NewJiraClient(&externalsvc.JiraOptions{BaseURL: srv.URL})
+	require.NoError(t, err)
+
+	// create the failer, that will introduced forced errors
+	failer := &TestJiraFailer{
+		FailCallCountModulo: 3,
+		AlwaysFailCVEs:      []string{"CVE-2020-1234"},
+		JiraClient:          client,
+	}
+
+	// create the Jira job with that failer-wrapped client
+	jira := &Jira{
+		FleetURL:   "http://example.com",
+		Datastore:  ds,
+		Log:        kitlog.NewNopLogger(),
+		JiraClient: failer,
+	}
+
+	var failedIndices []int
+	cves := []string{"CVE-2018-1234", "CVE-2019-1234", "CVE-2020-1234", "CVE-2021-1234"}
+	for i := 0; i < 10; i++ {
+		cve := cves[i%len(cves)]
+		err := jira.Run(context.Background(), json.RawMessage(fmt.Sprintf(`{"cve":%q}`, cve)))
+		if err != nil {
+			failedIndices = append(failedIndices, i)
+		}
+	}
+
+	// want indices:
+	// 2: always failing CVE
+	// 5: modulo
+	// 6: CVE
+	// 8: modulo
+	require.Equal(t, []int{2, 5, 6, 8}, failedIndices)
+}