Fleet UI: Live query UI and export results tables include all columns returned (#13428)

changes/12476-ui-export-shows-all-columns

@@ -0,0 +1 @@
+- Bug fix: Live query UI and Export data tables show all returned columns

frontend/pages/hosts/ManageHostsPage/ManageHostsPage.tsx

@@ -1285,7 +1285,6 @@ const ManageHostsPage = ({
       { [`${baseClass}__status-dropdown-sandbox`]: isSandboxMode }
     );
 
-    console.log("status", status);
     return (
       <div className={`${baseClass}__filter-dropdowns`}>
         <Dropdown

frontend/pages/policies/PolicyPage/components/QueryResults/QueryResults.tsx

@@ -1,12 +1,15 @@
 import React, { useState, useContext } from "react";
 import { Tab, Tabs, TabList, TabPanel } from "react-tabs";
 import classnames from "classnames";
-import { format } from "date-fns";
 import FileSaver from "file-saver";
 import { get } from "lodash";
 import { PolicyContext } from "context/policy";
 
-import convertToCSV from "utilities/convert_to_csv";
+import {
+  generateCSVFilename,
+  generateCSVPolicyResults,
+  generateCSVPolicyErrors,
+} from "utilities/generate_csv";
 import { ICampaign } from "interfaces/campaign";
 import { ITarget } from "interfaces/target";
 
@@ -71,14 +74,13 @@ const QueryResults = ({
             host.query_results && host.query_results.length ? "yes" : "no",
         };
       });
-      const csv = convertToCSV(hostsExport);
-      const formattedTime = format(new Date(), "MM-dd-yy hh-mm-ss");
-      const filename = `${policyName || CSV_TITLE} (${formattedTime}).csv`;
-      const file = new global.window.File([csv], filename, {
-        type: "text/csv",
-      });
 
-      FileSaver.saveAs(file);
+      FileSaver.saveAs(
+        generateCSVPolicyResults(
+          hostsExport,
+          generateCSVFilename(`${policyName || CSV_TITLE} - Results`)
+        )
+      );
     }
   };
 
@@ -86,17 +88,12 @@ const QueryResults = ({
     evt.preventDefault();
 
     if (errors) {
-      const csv = convertToCSV(errors);
-
-      const formattedTime = format(new Date(), "MM-dd-yy hh-mm-ss");
-      const filename = `${
-        policyName || CSV_TITLE
-      } Errors (${formattedTime}).csv`;
-      const file = new global.window.File([csv], filename, {
-        type: "text/csv",
-      });
-
-      FileSaver.saveAs(file);
+      FileSaver.saveAs(
+        generateCSVPolicyErrors(
+          errors,
+          generateCSVFilename(`${policyName || CSV_TITLE} - Errors`)
+        )
+      );
     }
   };
 

frontend/pages/queries/QueryPage/components/QueryResults/QueryResults.tsx

@@ -2,12 +2,14 @@ import React, { useState, useContext, useEffect, useCallback } from "react";
 import { Row, Column } from "react-table";
 import { Tab, Tabs, TabList, TabPanel } from "react-tabs";
 import classnames from "classnames";
-import { format } from "date-fns";
 import FileSaver from "file-saver";
 import { QueryContext } from "context/query";
 import { useDebouncedCallback } from "use-debounce";
 
-import convertToCSV from "utilities/convert_to_csv";
+import {
+  generateCSVFilename,
+  generateCSVQueryResults,
+} from "utilities/generate_csv";
 import { ICampaign } from "interfaces/campaign";
 import { ITarget } from "interfaces/target";
 
@@ -24,6 +26,7 @@ import generateResultsTableHeaders from "./QueryResultsTableConfig";
 interface IQueryResultsProps {
   campaign: ICampaign;
   isQueryFinished: boolean;
+  queryName?: string;
   onRunQuery: () => void;
   onStopQuery: (evt: React.MouseEvent<HTMLButtonElement>) => void;
   setSelectedTargets: (value: ITarget[]) => void;
@@ -32,41 +35,16 @@ interface IQueryResultsProps {
 }
 
 const baseClass = "query-results";
-const CSV_QUERY_TITLE = "Query Results";
+const CSV_TITLE = "New Query";
 const NAV_TITLES = {
   RESULTS: "Results",
   ERRORS: "Errors",
 };
 
-const reorderCSVFields = (fields: string[]) => {
-  const result = fields.filter((field) => field !== "host_display_name");
-  result.unshift("host_display_name");
-
-  return result;
-};
-
-const generateExportCSVFile = (rows: Row[], filename: string) => {
-  return new global.window.File(
-    [
-      convertToCSV(
-        rows.map((r) => r.original),
-        reorderCSVFields
-      ),
-    ],
-    filename,
-    {
-      type: "text/csv",
-    }
-  );
-};
-
-const generateExportFilename = (descriptor: string) => {
-  return `${descriptor} (${format(new Date(), "MM-dd-yy hh-mm-ss")}).csv`;
-};
-
 const QueryResults = ({
   campaign,
   isQueryFinished,
+  queryName,
   onRunQuery,
   onStopQuery,
   setSelectedTargets,
@@ -82,10 +60,8 @@ const QueryResults = ({
   const [showQueryModal, setShowQueryModal] = useState(false);
   const [filteredResults, setFilteredResults] = useState<Row[]>([]);
   const [filteredErrors, setFilteredErrors] = useState<Row[]>([]);
-  const [tableHeaders, setTableHeaders] = useState<null | Column[]>([]);
-  const [errorTableHeaders, setErrorTableHeaders] = useState<null | Column[]>(
-    []
-  );
+  const [tableHeaders, setTableHeaders] = useState<Column[]>([]);
+  const [errorTableHeaders, setErrorTableHeaders] = useState<Column[]>([]);
   const [queryResultsForTableRender, setQueryResultsForTableRender] = useState(
     queryResults
   );
@@ -102,32 +78,43 @@ const QueryResults = ({
     { maxWait: 2000 }
   );
 
+  // This is throwing an error not to use hook within a useEffect
   useEffect(() => {
     debounceQueryResults(queryResults);
   }, [queryResults, debounceQueryResults]);
 
-  // set tableHeaders when initial results come in
-  // instead of memoizing tableHeaders, since we know the conditions exactly under which we want to
-  // set these
   useEffect(() => {
-    if (tableHeaders?.length === 0 && !!queryResults?.length) {
-      setTableHeaders(generateResultsTableHeaders(queryResults));
+    if (queryResults && queryResults.length > 0) {
+      const generatedTableHeaders = generateResultsTableHeaders(queryResults);
+      // Update tableHeaders if new headers are found
+      if (generatedTableHeaders !== tableHeaders) {
+        setTableHeaders(generatedTableHeaders);
+      }
     }
-  }, [tableHeaders, queryResults]);
+  }, [queryResults]); // Cannot use tableHeaders as it will cause infinite loop with setTableHeaders
 
   useEffect(() => {
     if (errorTableHeaders?.length === 0 && !!errors?.length) {
       setErrorTableHeaders(generateResultsTableHeaders(errors));
+
+      if (errorTableHeaders && errorTableHeaders.length > 0) {
+        const generatedErrorTableHeaders = generateResultsTableHeaders(errors);
+
+        // Update errorTableHeaders if new headers are found
+        if (generatedErrorTableHeaders !== tableHeaders) {
+          setErrorTableHeaders(generatedErrorTableHeaders);
+        }
+      }
     }
-  }, [errorTableHeaders, errors]);
+  }, [errors]); // Cannot use errorTableHeaders as it will cause infinite loop with setErrorTableHeaders
 
   const onExportQueryResults = (evt: React.MouseEvent<HTMLButtonElement>) => {
     evt.preventDefault();
-
     FileSaver.saveAs(
-      generateExportCSVFile(
+      generateCSVQueryResults(
         filteredResults,
-        generateExportFilename(CSV_QUERY_TITLE)
+        generateCSVFilename(`${queryName || CSV_TITLE} - Results`),
+        tableHeaders
       )
     );
   };
@@ -136,9 +123,10 @@ const QueryResults = ({
     evt.preventDefault();
 
     FileSaver.saveAs(
-      generateExportCSVFile(
+      generateCSVQueryResults(
         filteredErrors,
-        generateExportFilename(`${CSV_QUERY_TITLE} Errors`)
+        generateCSVFilename(`${queryName || CSV_TITLE} - Errors`),
+        errorTableHeaders
       )
     );
   };

frontend/pages/queries/QueryPage/components/QueryResults/QueryResultsTableConfig.tsx

@@ -47,28 +47,28 @@ const _unshiftHostname = (headers: IDataColumn[]) => {
   return newHeaders;
 };
 
-const generateResultsTableHeaders = (results: unknown[]): Column[] => {
-  // Table headers are derived from the shape of the first result.
-  // Note: It is possible that results may vary from the shape of the first result.
-  // For example, different versions of osquery may have new columns in a table
-  // However, this is believed to be a very unlikely scenario and there have been
-  // no reported issues.
-  const shape = results[0];
-  const keys =
-    shape && typeof shape === "object" && isPlainObject(shape)
-      ? Object.keys(shape)
-      : [];
-  const headers = keys.map((key) => {
+const generateResultsTableHeaders = (results: any[]): Column[] => {
+  /* Results include an array of objects, each representing a table row
+  Each key value pair in an object represents a column name and value
+  To create headers, use JS set to create an array of all unique column names */
+  const uniqueColumnNames = Array.from(
+    results.reduce(
+      (s, o) => Object.keys(o).reduce((t, k) => t.add(k), s),
+      new Set() // Set prevents listing duplicate headers
+    )
+  );
+
+  const headers = uniqueColumnNames.map((key) => {
     return {
-      id: key,
-      title: key,
+      id: key as string,
+      title: key as string,
       Header: (headerProps: IHeaderProps) => (
         <HeaderCell
           value={headerProps.column.title || headerProps.column.id}
           isSortedDesc={headerProps.column.isSortedDesc}
         />
       ),
-      accessor: key,
+      accessor: key as string,
       Cell: (cellProps: ICellProps) => cellProps?.cell?.value || null,
       Filter: DefaultColumnFilter,
       // filterType: "text",

frontend/pages/queries/QueryPage/screens/RunQuery.tsx

@@ -205,6 +205,7 @@ const RunQuery = ({
       isQueryFinished={isQueryFinished}
       setSelectedTargets={setSelectedTargets}
       goToQueryEditor={goToQueryEditor}
+      queryName={storedQuery?.name}
       targetsTotalCount={targetsTotalCount}
     />
   );

frontend/utilities/convert_to_csv/convert_to_csv.tests.ts

@@ -13,7 +13,7 @@ const objArray = [
 
 describe("convertToCSV - utility", () => {
   it("converts an array of objects to CSV format", () => {
-    expect(convertToCSV(objArray)).toEqual(
+    expect(convertToCSV({ objArray })).toEqual(
       '"first_name","last_name"\n"Mike","Stone"\n"Paul","Simon"'
     );
   });

frontend/utilities/convert_to_csv/index.ts

@@ -1,12 +1,25 @@
-import { keys } from "lodash";
+import { ICampaignError } from "interfaces/campaign";
+import { Row, Column } from "react-table";
 
 const defaultFieldSortFunc = (fields: string[]) => fields;
 
-const convertToCSV = (
-  objArray: any[],
-  fieldSortFunc = defaultFieldSortFunc
-) => {
-  const fields = fieldSortFunc(keys(objArray[0]));
+interface ConvertToCSV {
+  objArray: any; // TODO: typing
+  fieldSortFunc?: (fields: string[]) => string[];
+  tableHeaders?: any[]; // TODO: typing
+}
+
+const convertToCSV = ({
+  objArray,
+  fieldSortFunc = defaultFieldSortFunc,
+  tableHeaders,
+}: ConvertToCSV) => {
+  const tableHeadersStrings: string[] = tableHeaders
+    ? tableHeaders.map((header: { id: string }) => header.id) // TODO: typing
+    : Object.keys(objArray[0]);
+
+  const fields = fieldSortFunc(tableHeadersStrings);
+
   // TODO: Remove after v5 when host_hostname is removed rom API response.
   const hostNameIndex = fields.indexOf("host_hostname");
   if (hostNameIndex >= 0) {
@@ -14,7 +27,8 @@ const convertToCSV = (
   }
   // Remove end
   const jsonFields = fields.map((field) => JSON.stringify(field));
-  const rows = objArray.map((row) => {
+  const rows = objArray.map((row: any) => {
+    // TODO: typing
     return fields.map((field) => JSON.stringify(row[field])).join(",");
   });
 

frontend/utilities/generate_csv/index.ts

@@ -0,0 +1,63 @@
+import convertToCSV from "utilities/convert_to_csv";
+import { Row, Column } from "react-table";
+import { ICampaignError } from "interfaces/campaign";
+import { format } from "date-fns";
+
+const reorderCSVFields = (tableHeaders: string[]) => {
+  const result = tableHeaders.filter((field) => field !== "host_display_name");
+  result.unshift("host_display_name");
+
+  return result;
+};
+
+export const generateCSVFilename = (descriptor: string) => {
+  return `${descriptor} (${format(new Date(), "MM-dd-yy hh-mm-ss")}).csv`;
+};
+
+// Query results and query errors
+export const generateCSVQueryResults = (
+  rows: Row[],
+  filename: string,
+  tableHeaders: Column[] | string[]
+) => {
+  return new global.window.File(
+    [
+      convertToCSV({
+        objArray: rows.map((r) => r.original),
+        fieldSortFunc: reorderCSVFields,
+        tableHeaders,
+      }),
+    ],
+    filename,
+    {
+      type: "text/csv",
+    }
+  );
+};
+
+// Policy results only
+export const generateCSVPolicyResults = (
+  rows: { host: string; status: string }[],
+  filename: string
+) => {
+  return new global.window.File([convertToCSV({ objArray: rows })], filename, {
+    type: "text/csv",
+  });
+};
+
+// Policy errors only
+export const generateCSVPolicyErrors = (
+  rows: ICampaignError[],
+  filename: string
+) => {
+  return new global.window.File([convertToCSV({ objArray: rows })], filename, {
+    type: "text/csv",
+  });
+};
+
+export default {
+  generateCSVFilename,
+  generateCSVQueryResults,
+  generateCSVPolicyResults,
+  generateCSVPolicyErrors,
+};