Web UI no longer gives an error when deleting a large number of hosts. (#14896)

After 30 seconds, the 'Delete host' modal closes and the delete
operation continues in the background. The following text has been added
to the modal when deleting 500 or more hosts: "When deleting a large
volume of hosts, it may take some time for this change to be reflected
in the UI."

#14097 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [x] Manual QA for all new/changed functionality
- [x] Backend test added

changes/14097-deleting-large-number-of-hosts

@@ -0,0 +1,4 @@
+Web UI no longer gives an error when deleting a large number of hosts.
+
+After 30 seconds, the 'Delete host' modal closes and the delete operation continues in the background.
+The following text has been added to the modal when deleting 500 or more hosts: "When deleting a large volume of hosts, it may take some time for this change to be reflected in the UI."

frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx

@@ -1179,6 +1179,7 @@ const ManageHostsPage = ({
       onSubmit={onDeleteHostSubmit}
       onCancel={toggleDeleteHostModal}
       isAllMatchingHostsSelected={isAllMatchingHostsSelected}
+      hostsCount={hostsCount}
       isUpdating={isUpdatingHosts}
     />
   );

frontend/pages/hosts/components/DeleteHostModal/DeleteHostModal.tsx

@@ -13,6 +13,8 @@ interface IDeleteHostModalProps {
   isAllMatchingHostsSelected?: boolean;
   /** Manage host page only */
   selectedHostIds?: number[];
+  /** Manage host page only */
+  hostsCount?: number;
   /** Host details page only */
   hostName?: string;
   isUpdating: boolean;
@@ -23,6 +25,7 @@ const DeleteHostModal = ({
   onCancel,
   isAllMatchingHostsSelected,
   selectedHostIds,
+  hostsCount,
   hostName,
   isUpdating,
 }: IDeleteHostModalProps): JSX.Element => {
@@ -34,6 +37,12 @@ const DeleteHostModal = ({
     }
     return hostName;
   };
+  const largeVolumeText = (): string => {
+    if (selectedHostIds && isAllMatchingHostsSelected && hostsCount && hostsCount >= 500) {
+      return " When deleting a large volume of hosts, it may take some time for this change to be reflected in the UI."
+    }
+    return ""
+  }
 
   return (
     <Modal
@@ -44,7 +53,7 @@ const DeleteHostModal = ({
     >
       <form className={`${baseClass}__form`}>
         <p>
-          This action will delete <b>{hostText()}</b> from your Fleet instance.
+          This action will delete <b>{hostText()}</b> from your Fleet instance.{largeVolumeText()}
         </p>
         <p>If the hosts come back online, they will automatically re-enroll.</p>
         <p>

server/service/hosts.go

@@ -161,6 +161,12 @@ func (svc *Service) ListHosts(ctx context.Context, opt fleet.HostListOptions) ([
 // Delete Hosts
 /////////////////////////////////////////////////////////////////////////////////
 
+// These values are modified during testing.
+var (
+	deleteHostsTimeout           = 30 * time.Second
+	deleteHostsSkipAuthorization = false
+)
+
 type deleteHostsRequest struct {
 	IDs     []uint `json:"ids"`
 	Filters struct {
@@ -172,11 +178,15 @@ type deleteHostsRequest struct {
 }
 
 type deleteHostsResponse struct {
-	Err error `json:"error,omitempty"`
+	Err        error `json:"error,omitempty"`
+	StatusCode int   `json:"-"`
 }
 
 func (r deleteHostsResponse) error() error { return r.Err }
 
+// Status implements statuser interface to send out custom HTTP success codes.
+func (r deleteHostsResponse) Status() int { return r.StatusCode }
+
 func deleteHostsEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) {
 	req := request.(*deleteHostsRequest)
 	listOpt := fleet.HostListOptions{
@@ -186,11 +196,33 @@ func deleteHostsEndpoint(ctx context.Context, request interface{}, svc fleet.Ser
 		StatusFilter: req.Filters.Status,
 		TeamFilter:   req.Filters.TeamID,
 	}
-	err := svc.DeleteHosts(ctx, req.IDs, listOpt, req.Filters.LabelID)
-	if err != nil {
-		return deleteHostsResponse{Err: err}, nil
+
+	// Since bulk deletes can take a long time, after DeleteHostsTimeout, we will return a 202 (Accepted) status code
+	// and allow the delete operation to proceed.
+	var err error
+	deleteDone := make(chan bool, 1)
+	ctx = context.WithoutCancel(ctx) // to make sure DB operations don't get killed after we return a 202
+	go func() {
+		err = svc.DeleteHosts(ctx, req.IDs, listOpt, req.Filters.LabelID)
+		if err != nil {
+			// logging the error for future debug in case we already sent http.StatusAccepted
+			logging.WithErr(ctx, err)
+		}
+		deleteDone <- true
+	}()
+	select {
+	case <-deleteDone:
+		if err != nil {
+			return deleteHostsResponse{Err: err}, nil
+		}
+		return deleteHostsResponse{StatusCode: http.StatusOK}, nil
+	case <-time.After(deleteHostsTimeout):
+		if deleteHostsSkipAuthorization {
+			// Only called during testing.
+			svc.(validationMiddleware).Service.(*Service).authz.SkipAuthorization(ctx)
+		}
+		return deleteHostsResponse{StatusCode: http.StatusAccepted}, nil
 	}
-	return deleteHostsResponse{}, nil
 }
 
 func (svc *Service) DeleteHosts(ctx context.Context, ids []uint, opts fleet.HostListOptions, lid *uint) error {
@@ -198,6 +230,10 @@ func (svc *Service) DeleteHosts(ctx context.Context, ids []uint, opts fleet.Host
 		return err
 	}
 
+	if len(ids) == 0 && lid == nil && opts.Empty() {
+		return &fleet.BadRequestError{Message: "list of ids or filters must be specified"}
+	}
+
 	if len(ids) > 0 && (lid != nil || !opts.Empty()) {
 		return &fleet.BadRequestError{Message: "Cannot specify a list of ids and filters at the same time"}
 	}

server/service/integration_core_test.go

@@ -963,6 +963,44 @@ func (s *integrationTestSuite) TestBulkDeleteHostByIDs() {
 	require.NoError(t, err)
 }
 
+func (s *integrationTestSuite) TestBulkDeleteHostByIDsWithTimeout() {
+	t := s.T()
+
+	hosts := s.createHosts(t, "debian")
+
+	req := deleteHostsRequest{
+		IDs: []uint{hosts[0].ID},
+	}
+	resp := deleteHostsResponse{}
+	originalTimeout := deleteHostsTimeout
+	deleteHostsTimeout = 0
+	deleteHostsSkipAuthorization = true
+	defer func() {
+		deleteHostsTimeout = originalTimeout
+		deleteHostsSkipAuthorization = false
+	}()
+	s.DoJSON("POST", "/api/latest/fleet/hosts/delete", req, http.StatusAccepted, &resp)
+
+	// Make sure the host was actually deleted.
+	deleteDone := make(chan bool)
+	go func() {
+		for {
+			_, err := s.ds.Host(context.Background(), hosts[0].ID)
+			if err != nil {
+				deleteDone <- true
+				break
+			}
+		}
+	}()
+	select {
+	case <-deleteDone:
+		return
+	case <-time.After(2 * time.Second):
+		t.Log("http.StatusAccepted (202) means that delete should continue in the background, but we did not see the host deleted after 2 seconds.")
+		t.Error("Timeout: delete did not occur.")
+	}
+}
+
 func (s *integrationTestSuite) createHosts(t *testing.T, platforms ...string) []*fleet.Host {
 	var hosts []*fleet.Host
 	if len(platforms) == 0 {