From f3926c4677a272ec2d64bf4ba58f53060c3fc3c8 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 13 Apr 2022 09:17:02 -0400 Subject: [PATCH] Introduce forced failures for the Jira client. (#5088) --- cmd/fleet/cron.go | 27 ++++++++++- server/worker/jira_failer.go | 53 +++++++++++++++++++++ server/worker/jira_failer_test.go | 77 +++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 server/worker/jira_failer.go create mode 100644 server/worker/jira_failer_test.go diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 169378f365..a69a7bbfb9 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -3,6 +3,8 @@ package main import ( "context" "os" + "strconv" + "strings" "time" "github.com/fleetdm/fleet/v4/server/config" @@ -403,6 +405,24 @@ func cronWorker( } w.Register(jira) + // create a JiraClient wrapper to introduce forced failures if configured + // to do so via the environment variable. + var failerClient *worker.TestJiraFailer + if forcedFailures := os.Getenv("FLEET_JIRA_CLIENT_FORCED_FAILURES"); forcedFailures != "" { + // format is ";,,,..." + parts := strings.Split(forcedFailures, ";") + if len(parts) == 2 { + mod, _ := strconv.Atoi(parts[0]) + cves := strings.Split(parts[1], ",") + if mod > 0 || len(cves) > 0 { + failerClient = &worker.TestJiraFailer{ + FailCallCountModulo: mod, + AlwaysFailCVEs: cves, + } + } + } + } + ticker := time.NewTicker(10 * time.Second) for { select { @@ -461,7 +481,12 @@ func cronWorker( // safe to update the Jira worker as it is not used concurrently jira.FleetURL = appConfig.ServerSettings.ServerURL - jira.JiraClient = client + if failerClient != nil && strings.Contains(jira.FleetURL, "fleetdm") { + failerClient.JiraClient = client + jira.JiraClient = failerClient + } else { + jira.JiraClient = client + } workCtx, cancel := context.WithTimeout(ctx, lockDuration) if err := w.ProcessJobs(workCtx); err != nil { diff --git a/server/worker/jira_failer.go b/server/worker/jira_failer.go new file mode 100644 index 0000000000..18d50962b7 --- /dev/null +++ b/server/worker/jira_failer.go @@ -0,0 +1,53 @@ +package worker + +import ( + "context" + "fmt" + "strings" + + jira "github.com/andygrunwald/go-jira" +) + +// TestJiraFailer is an implementation of the JiraClient interface that wraps +// another JiraClient and introduces forced failures so that error-handling +// logic can be tested at scale in a real environment (e.g. in the load-testing +// environment). +type TestJiraFailer struct { + // FailCallCountModulo is the number of calls to execute normally vs + // forcing a failure. In other words, it will force a failure every time + // callCounts % FailCallCountModulo == 0. If it is <= 0, no forced failure is + // introduced based on call counts. + FailCallCountModulo int + + // AlwaysFailCVEs is the list of CVEs for which a failure will always be + // forced, so that those CVEs never succeed in creating a Jira ticket. + AlwaysFailCVEs []string + + // JiraClient is the wrapped Jira client to use for normal calls, when no + // forced failure is inserted. + JiraClient JiraClient + + callCounts int +} + +// CreateIssue implements the JiraClient and introduces a forced failure if +// required, otherwise it returns the result of calling +// f.JiraClient.CreateIssue with the provided arguments. +func (f *TestJiraFailer) CreateIssue(ctx context.Context, issue *jira.Issue) (*jira.Issue, error) { + f.callCounts++ + + if issue.Fields != nil && issue.Fields.Summary != "" { + s := issue.Fields.Summary + for _, cve := range f.AlwaysFailCVEs { + if strings.Contains(s, cve) { + return nil, fmt.Errorf("always failing CVE %q", cve) + } + } + } + + if f.FailCallCountModulo > 0 && f.callCounts%f.FailCallCountModulo == 0 { + return nil, fmt.Errorf("failing due to FailCallCountModulo: callCount=%d", f.callCounts) + } + + return f.JiraClient.CreateIssue(ctx, issue) +} diff --git a/server/worker/jira_failer_test.go b/server/worker/jira_failer_test.go new file mode 100644 index 0000000000..41d4c40e35 --- /dev/null +++ b/server/worker/jira_failer_test.go @@ -0,0 +1,77 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/service/externalsvc" + kitlog "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" +) + +func TestTestJiraFailer(t *testing.T) { + ds := new(mock.Store) + ds.HostsByCVEFunc = func(ctx context.Context, cve string) ([]*fleet.HostShort, error) { + return []*fleet.HostShort{{ID: 1, Hostname: "test"}}, nil + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + w.Write([]byte(` +{ + "id": "10000", + "key": "ED-24", + "self": "https://your-domain.atlassian.net/rest/api/2/issue/10000", + "transition": { + "status": 200, + "errorCollection": { + "errorMessages": [], + "errors": {} + } + } +}`)) + })) + defer srv.Close() + + // create the real client, that will never fail + client, err := externalsvc.NewJiraClient(&externalsvc.JiraOptions{BaseURL: srv.URL}) + require.NoError(t, err) + + // create the failer, that will introduced forced errors + failer := &TestJiraFailer{ + FailCallCountModulo: 3, + AlwaysFailCVEs: []string{"CVE-2020-1234"}, + JiraClient: client, + } + + // create the Jira job with that failer-wrapped client + jira := &Jira{ + FleetURL: "http://example.com", + Datastore: ds, + Log: kitlog.NewNopLogger(), + JiraClient: failer, + } + + var failedIndices []int + cves := []string{"CVE-2018-1234", "CVE-2019-1234", "CVE-2020-1234", "CVE-2021-1234"} + for i := 0; i < 10; i++ { + cve := cves[i%len(cves)] + err := jira.Run(context.Background(), json.RawMessage(fmt.Sprintf(`{"cve":%q}`, cve))) + if err != nil { + failedIndices = append(failedIndices, i) + } + } + + // want indices: + // 2: always failing CVE + // 5: modulo + // 6: CVE + // 8: modulo + require.Equal(t, []int{2, 5, 6, 8}, failedIndices) +}