extendedDescription:'This script is where all of the data in the vulnerability dashboard comes from, and is designed be run hourly via scheduled or cron job.',
inputs:{
dry:{
description:'Do a dry run instead of actually modifying the database?',
type:'boolean',
defaultsTo:false
},
},
fn:asyncfunction({dry}){
// [?] Wondering where all the performance notes and commented-out methods of processing API data went?
// You can find them on this commit -» https://github.com/fleetdm/fleet-vulnerability-dashboard/blob/1c58578c149d97307ae288000c80257b29bb3126/scripts/update-reports.js
thrownewError(`Invalid sails.config.custom.updateReportsFleetApiPageSize value. Please change sails.config.custom.updateReportsFleetApiPageSize to be a number. (typeof sails.config.custom.updateReportsFleetApiPageSize: ${typeofsails.config.custom.updateReportsFleetApiPageSize})`);
// Build a dictionary where every existing VulnerabilityInstall record has a unique string as it's key that we'll use to check if a install records we're processing exists already.
letexistingVulnInstallsByHostAndVulnIDs={};
// Build a second dictionary to track existing VulnerabilityInstall records that are not included in the Fleet API response this run.
letmissingVulnInstallsByIds={};
letvulnInstallsResolvedVersionsByApids={};
// Do the same thing for CriticalInstalls.
letexistingCriticalInstallsByHostIDs={};
letcompliantVersionsApids=[];
letmissingCriticalInstallsByHostIDs={};
// Create an array to store the versions of compliant microsoft office software.Add the versions of compliant Microsoft office installs to an array.
// We'll add the same unique string that we use for the existingVulnInstallsByHostAndVulnIDs dictionary as the key, but the values will be the database ID of the VulnerabilityInstall.
missingVulnInstallsByIds[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}|${$vulnInstall.host}`]=$vulnInstall.id;// « ex {'123615|1998|4': 8323, '123615|1998|8': 8324, '123615|1998|9': 8325, ...}
// Get Operating system information from the Fleet instance, we create these records first so we can reference them when saving Host information to the database.
// If an operating system returned in the /os_versions API response is missing a version, we'll log a warning, but we'll still create a record for it with 'N/A' set as the version.
// This is so we are still able to create host records for hosts with this operating system installed. (The ID of an operating system record is a required value for host records)
loggedWarningsFromThisScriptRun.push(`An operating system (name: ${os.name}) returned in the response from the /os_versions endpoint is missing a 'version'. This operating system will be reported as having "N/A" as the version. Operating system without a version:`,os);
returnnewError(`When creating new OperatingSystem records from operating systems returned from the Fleet API, an error occured. Full Error: ${error}`);
returnnewError(`When sending a request to the '/api/v1/fleet/hosts' API endpoint to get information about the hosts on this instance, the Fleet instance returned a 404 response when we expected it to return an array of ${HOSTS_PAGE_SIZE} host(s).\n Response from Fleet instance: ${error.raw.body}`);
});
// sails.log.warn(`got a response for page ${hostsPage} for ${ware.name}.`);
if(!responseData.hosts){// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
sails.log.verbose(`When building host records from the Fleet API, a host's (FleetApid: ${host.id}) os_version (os_version: ${host.os_version}) could not be matched to an operating system returned in the API response from the /os_versions endpoint. Host with missing operating system:`,host);
teamApid:host.team_id!==null?host.team_id:0,// Note: If the host is not a member of any team, we'll set this to 0
fleetApid:host.id,
hardwareSerialNumber:host.hardware_serial!==''?host.hardware_serial:'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
// If the hosts API returns a 404 response for a software item that was returned from in the list of vulnerable software, we'll log a warning and remove this software from the list of software.
loggedWarningsFromThisScriptRun.push(`When processing vulnerable software, a request to the '/hosts' endpoint to get a filtered array of hosts with ${ware.name}${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Impact: If this vulnerable software was previously processed, the database record(s) for it will be marked as uninstalled. If it shows up in a future run of this script, a new database record will be created.`);
vulnerableWaresWithNoHostInformation.push(ware);// Add this software to the vulnerableWaresWithNoHostInformation array, these will be removed before we create and update database records.
return{};// Return an empty object. This will let the script continue without information about this software.
sails.log.verbose(`When building host records from the Fleet API, a host's (FleetApid: ${host.id}) os_version (os_version: ${host.os_version}) could not be matched to an operating system returned in the API response from the /os_versions endpoint. Host with missing operating system:`,host);
continue;
}
}
byHostFleetApidsSeenInLatestFleetScan[host.id]={
displayName:_.trim(host.display_name.replace(/,/g,'')),// Trim leading and trailing whitespace, and remove commas from Host names.
teamApid:host.team_id!==null?host.team_id:0,// Note: If the host is not a member of any team, we'll set this to 0
fleetApid:host.id,
hardwareSerialNumber:host.hardware_serial!==''?host.hardware_serial:'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
uuid:host.uuid,
operatingSystem:hostsOperatingSystem.id,
};//∞
}
}).intercept((err)=>{
sails.log.error('An error occurred in side the iteratee of a flow control function. Due to an unresolved issue, these errors can be hard to dissect without code changes. Just in case, this log message seeks to dump as much info as possible!\n','err',err,'\nerr.raw',err.raw);
// // Since .raw isn't displayed in output again in Node ≥12, try to make the error better.
// Build a formatted, filtered set of the vulnerabilities returned by this scan.
// These will be analyzed below and potentially written to the database.
letpotentiallyNewVulnRecords=[];
letcveIdsBySoftwareVersionApid={};// « for use below when creating vuln installation records
for(letwareofvulnerableWares){
softwareInfoForVulnInstalls[ware.id]={
softwareName:ware.name.replace(/,/g,''),// Remove any commas in software names
versionName:ware.version!==''?ware.version.replace(/,/g,''):'N/A',// Remove any commas in version names. if a version name is missing, this will be added as 'N/A'
// sails.log.warn('Unrecognized CVE detected in Fleet:' + require('util').inspect(fleetVuln, {depth:null}), 'for a software with the ID of '+ware.id+'. This means that the data being used to hydrate the vulnerability data with additional info such as publish date is missing this particular CVE, even though it was reported in the results from scanning with Fleet. For consistency, excluding this vulnerability from the Fleet scan results as if it was not detected.');
}elseif(!fleetVuln.cvss_score){
// sails.log.verbose('Invalid CVSS score detected by Fleet:' + require('util').inspect(fleetVuln, {depth:null}), 'This might mean that the vulnerability is undergoing reanlysis. For now, this vulnerability will be treated as if it was not detected in the Fleet scan results.');
probabilityOfExploit:fleetVuln.epss_probability!==null?fleetVuln.epss_probability:0,// If the Fleet server sends this value as null, we'll set this to be 0.
sails.log.warn(`Dry run: ${newlyDiscoveredCveIds.length} newly discovered vulnerabilities (CVEs) are available and updates to ${newlyUpdatedVulns.length} vulnerabilities were found. (Fleet returned them in the API.)`);
}else{
sails.log(`Detected ${newlyDiscoveredCveIds.length} new vulnerabilities and updates to ${newlyUpdatedVulns.length} vulnerabilities were found.. Saving...`);
}elseif(thisVulnInstall.resolvedInVersion){// If this vulnerable software has a resolvedInVersion value, we'll check to see if it is what we previosly recorded.
// If a record for this vuln install exists, we'll check the affectedVersionName value of the existing record to see if it should be updated.
sails.log.warn(`Dry run: ${potentialVulnInstalls.length} potential vulnerability installs are available and ${potentialVulnInstallUpdates.length} updates are available for existing vulnerability installs.`);
}else{
sails.log(`Detected ${potentialVulnInstalls.length} changes to software installations and ${potentialVulnInstallUpdates.length} updates are available for existing vulnerability installs. Saving...`);
// If the hosts API returns a 404 response for a software item that was returned from in the list of critical software, we'll log a warning and remove this software from the list of software.
loggedWarningsFromThisScriptRun.push(`When processing critical software, a request to the '/hosts' endpoint to get a filtered array of hosts with ${ware.name}${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Impact: This software will be marked as uninstalled, and a new database record will be created if it shows up in a future run of this script.`);
criticalWaresWithNoHostInformation.push(ware);// Add this software to the criticalWaresWithNoHostInformation array, these will be removed before we create and update database records.
return{};// Return an empty object. This will let the script continue without information about this software.
sails.log.verbose(`When building host records from the Fleet API, a host's (FleetApid: ${host.id}) os_version (os_version: ${host.os_version}) could not be matched to an operating system returned in the API response from the /os_versions endpoint. Host with missing operating system:`,host);
teamApid:host.team_id!==null?host.team_id:0,// Note: If the host is not a member of any team, we'll set this to 0
fleetApid:host.id,
hardwareSerialNumber:host.hardware_serial!==''?host.hardware_serial:'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
uuid:host.uuid,
operatingSystem:hostsOperatingSystem.id,
};//∞
}
}).intercept((err)=>{
sails.log.error('An error occurred in side the iteratee of a flow control function. Due to an unresolved issue, these errors can be hard to dissect without code changes. Just in case, this log message seeks to dump as much info as possible!\n','err',err,'\nerr.raw',err.raw);
// // Since .raw isn't displayed in output again in Node ≥12, try to make the error better.
// (i.e. newly-fixed means previously-detected vulns that are no longer present)
letnewlyFixedVulns=[];
// > Note: This could be further optimized for huge datasets to update vulnerabilities gradually
// > as each batch is streamed. But it really only matters when many many vulns are fixed all at
// > once (i.e. `newlyFixedVulns`` overflowing).
awaitVulnerability.stream()
.meta({batchSize:1000})
.eachRecord(($vuln)=>{
// if (!Object.keys(byCveIdsSeenInLatestFleetScan).includes($vuln.cveId)) {
if(!byCveIdsSeenInLatestFleetScan[$vuln.cveId]){
newlyFixedVulns.push($vuln);
}
});//∞
if(dry){
sails.log.warn(`Dry run: ${newlyFixedVulns.length} previously-installed vulnerabilities were seemingly fixed. (Fleet did not return them in the API this time.)`);
}else{
// TODO: Verify that these are correct
sails.log(`${newlyFixedVulns.length} previously-installed vulnerabilities were seemingly fixed. (Fleet did not return them in the API this time.)`);
letvulnInstallRecordIdsThatNoLongerExistInFleetInstance=[];// « Create an array to store the database IDs of installs that Fleet did not return in the API response.
// This check is here to handles cases where vulnerable software is updated on a host, but the new version is affected by the same vulnerability as the old version.
for(letinstallToCheckinmissingVulnInstallsByIds){
if(missingVulnInstallsByIds[installToCheck]){// This will be set to false if this install was included in the API response from Fleet, otherwise it will be the Database ID of the VulnerabilityInstall record
sails.log.warn(`Dry run: ${vulnInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed vulnerable software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
}else{
sails.log(`${vulnInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed vulnerable software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
letcriticalInstallRecordIdsThatNoLongerExistInFleetInstance=[];// « Create an array to store the database IDs of installs that Fleet did not return in the API response.
// This check is here to handles cases where vulnerable software is updated on a host, but the new version is affected by the same vulnerability as the old version.
if(missingCriticalInstallsByHostIDs[installToCheck]){// This will be set to false if this install was included in the API response from Fleet, otherwise it will be the Database ID of the VulnerabilityInstall record
sails.log.warn(`Dry run: ${criticalInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed critical software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
}else{
sails.log(`${criticalInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed critical software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
sails.log.warn(`Dry run: ${osWithNoHosts.length} operating system(s) no longer in use found, and ${compliantOsWithNoHosts.length} operating system(s) that have been previously marked as compliant no longer reported by the Fleet instance user found.`);
}else{
sails.log(`${osWithNoHosts.length} operating system(s) no longer in use found, and ${compliantOsWithNoHosts.length} complaint operating systems no longer reported by the Fleet instance. Saving....`);
// Delete non-compliant operating system records that are no longer installed.
sails.log.warn(`Dry run: ${vulnerabilityRecordIdsWithNoAssociatedRecords.length} vulnerabilities affecting previously-enrolled hosts were found. (Fleet did not return them in the API this time.)`);
}else{
sails.log(`${vulnerabilityRecordIdsWithNoAssociatedRecords.length} vulnerabilities affecting previously-enrolled hosts were found. (Fleet did not return them in the API this time, and no associated Host or VulnerabilityInstall records were found.)`);
