mirror of
https://github.com/fleetdm/fleet
synced 2026-05-04 05:48:26 +00:00
e60abdd6e5
Related to: https://github.com/fleetdm/confidential/issues/7180 Changes: - Updated the `nativeQueryToFindVulnsWithNoAssociatedRecords` query to improve the performance of the update-reports script on large deployments
932 lines
62 KiB
JavaScript
932 lines
62 KiB
JavaScript
module.exports = {
|
|
|
|
friendlyName: 'Update reports',
|
|
|
|
description: 'Gather and process Fleet API data.',
|
|
extendedDescription: 'This script is where all of the data in the vulnerability dashboard comes from, and is designed be run hourly via scheduled or cron job.',
|
|
|
|
inputs: {
|
|
dry: {
|
|
description: 'Do a dry run instead of actually modifying the database?',
|
|
type: 'boolean',
|
|
defaultsTo: false
|
|
},
|
|
},
|
|
|
|
fn: async function ({ dry }) {
|
|
// [?] Wondering where all the performance notes and commented-out methods of processing API data went?
|
|
// You can find them on this commit -» https://github.com/fleetdm/fleet-vulnerability-dashboard/blob/1c58578c149d97307ae288000c80257b29bb3126/scripts/update-reports.js
|
|
let assert = require('assert');
|
|
let loggedWarningsFromThisScriptRun = [];
|
|
// ┌─┐┌─┐┌┬┐┬ ┬┌─┐┬─┐ ┌─┐┌┐┌┌┬┐ ┌─┐┬─┐┌─┐┌─┐┌─┐┌─┐┌─┐ ┌─┐┬ ┌─┐┌─┐┌┬┐ ┌─┐┌─┐┬ ┌┬┐┌─┐┌┬┐┌─┐
|
|
// │ ┬├─┤ │ ├─┤├┤ ├┬┘ ├─┤│││ ││ ├─┘├┬┘│ ││ ├┤ └─┐└─┐ ├┤ │ ├┤ ├┤ │ ├─┤├─┘│ ││├─┤ │ ├─┤
|
|
// └─┘┴ ┴ ┴ ┴ ┴└─┘┴└─ ┴ ┴┘└┘─┴┘ ┴ ┴└─└─┘└─┘└─┘└─┘└─┘ └ ┴─┘└─┘└─┘ ┴ ┴ ┴┴ ┴ ─┴┘┴ ┴ ┴ ┴ ┴
|
|
if (!sails.config.custom.fleetBaseUrl || !sails.config.custom.fleetApiToken) {
|
|
throw new Error('sails.config.custom.fleetBaseUrl and sails.config.custom.fleetApiToken must both be provided.');
|
|
}
|
|
let fleetBaseUrl = sails.config.custom.fleetBaseUrl;
|
|
let headers = {
|
|
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`
|
|
};
|
|
if (sails.config.custom.fleetApiOptionalCookie) {
|
|
headers['Cookie'] = sails.config.custom.fleetApiOptionalCookie;
|
|
}
|
|
|
|
// Keep track of the latest vulnerabilities, hosts, and software seen in the Fleet scan.
|
|
// We'll use these later to check if any records have gone missing.
|
|
let byCveIdsSeenInLatestFleetScan = {};
|
|
let byHostFleetApidsSeenInLatestFleetScan = {};
|
|
let byHostFleetApidsSeenInLatestCriticalSoftwareScan = {};
|
|
let softwareInfoForVulnInstalls = {};
|
|
// Now paginate over software items in the Fleet API, processing as we go along.
|
|
// (This is to avoid overflowing RAM on the server, in the antogonistic case where are just bookoos of vulnerabilities for some reason.)
|
|
let page = 0;
|
|
let SOFTWARE_VERSIONS_PAGE_SIZE = 100;
|
|
let HOSTS_PAGE_SIZE = 100;
|
|
// If sails.config.custom.updateReportsFleetApiPageSize is set, use that value for the page size for API requests.
|
|
if(sails.config.custom.updateReportsFleetApiPageSize) {
|
|
if(typeof sails.config.custom.updateReportsFleetApiPageSize !== 'number'){
|
|
throw new Error(`Invalid sails.config.custom.updateReportsFleetApiPageSize value. Please change sails.config.custom.updateReportsFleetApiPageSize to be a number. (typeof sails.config.custom.updateReportsFleetApiPageSize: ${typeof sails.config.custom.updateReportsFleetApiPageSize})`);
|
|
}
|
|
SOFTWARE_VERSIONS_PAGE_SIZE = sails.config.custom.updateReportsFleetApiPageSize;
|
|
HOSTS_PAGE_SIZE = sails.config.custom.updateReportsFleetApiPageSize;
|
|
}
|
|
let numVulnerableWaresProcessed = 0;
|
|
// Build two dictionaries from all existing VulnerabilityInstall and CriticalInstall records.
|
|
// This will be used for checking for existing records when we process data from the Fleet API.
|
|
let allKnownExistingVulnInstalls = await VulnerabilityInstall.find({select: ['host', 'vulnerability', 'fleetApid', 'uninstalledAt', 'resolvedInVersion']});
|
|
let allKnownExistingCriticalInstalls = await CriticalInstall.find({select: ['host', 'fleetApid', 'isCompliant']});
|
|
|
|
// Build a dictionary where every existing VulnerabilityInstall record has a unique string as it's key that we'll use to check if a install records we're processing exists already.
|
|
let existingVulnInstallsByHostAndVulnIDs = {};
|
|
// Build a second dictionary to track existing VulnerabilityInstall records that are not included in the Fleet API response this run.
|
|
let missingVulnInstallsByIds = {};
|
|
let vulnInstallsResolvedVersionsByApids = {};
|
|
// Do the same thing for CriticalInstalls.
|
|
let existingCriticalInstallsByHostIDs = {};
|
|
let compliantVersionsApids = [];
|
|
let missingCriticalInstallsByHostIDs = {};
|
|
// Create an array to store the versions of compliant microsoft office software.Add the versions of compliant Microsoft office installs to an array.
|
|
let compliantMicrosoftOfficeVersions = [];
|
|
for(let $vulnInstall of allKnownExistingVulnInstalls) {
|
|
if($vulnInstall.uninstalledAt === 0) {
|
|
// For every unresolved vulnerability, store the database ID of the VulnerabilityInstall record with a unique string as the key.
|
|
existingVulnInstallsByHostAndVulnIDs[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}|${$vulnInstall.host}`] = true;// « ex: {'140|56|2146':true, '135|2565|6729':true, ...}
|
|
// We'll add the same unique string that we use for the existingVulnInstallsByHostAndVulnIDs dictionary as the key, but the values will be the database ID of the VulnerabilityInstall.
|
|
missingVulnInstallsByIds[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}|${$vulnInstall.host}`] = $vulnInstall.id;// « ex {'123615|1998|4': 8323, '123615|1998|8': 8324, '123615|1998|9': 8325, ...}
|
|
}
|
|
vulnInstallsResolvedVersionsByApids[`${$vulnInstall.fleetApid}|${$vulnInstall.vulnerability}`] = $vulnInstall.resolvedInVersion;
|
|
}//∞
|
|
// And the same thing for CriticalInstalls
|
|
for(let $install of allKnownExistingCriticalInstalls) {
|
|
if($install.isCompliant){
|
|
if($install.softwareType === 'microsoftOffice'){
|
|
|
|
compliantMicrosoftOfficeVersions.push($install.versionName);
|
|
}
|
|
compliantVersionsApids.push($install.fleetApid);
|
|
}
|
|
existingCriticalInstallsByHostIDs[`${$install.fleetApid}|${$install.host}`] = true;// « ex: {'140|2146':true, '135|6729':true, ...}
|
|
missingCriticalInstallsByHostIDs[`${$install.fleetApid}|${$install.host}`] = $install.id;// « ex: {'140|2146':true, '135|6729':true, ...}
|
|
}//∞
|
|
|
|
|
|
// ██████╗ ███████╗ ██╗ ██╗███████╗██████╗ ███████╗██╗ ██████╗ ███╗ ██╗███████╗
|
|
// ██╔═══██╗██╔════╝ ██║ ██║██╔════╝██╔══██╗██╔════╝██║██╔═══██╗████╗ ██║██╔════╝
|
|
// ██║ ██║███████╗ ██║ ██║█████╗ ██████╔╝███████╗██║██║ ██║██╔██╗ ██║███████╗
|
|
// ██║ ██║╚════██║ ╚██╗ ██╔╝██╔══╝ ██╔══██╗╚════██║██║██║ ██║██║╚██╗██║╚════██║
|
|
// ╚██████╔╝███████║ ╚████╔╝ ███████╗██║ ██║███████║██║╚██████╔╝██║ ╚████║███████║
|
|
// ╚═════╝ ╚══════╝ ╚═══╝ ╚══════╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚══════╝
|
|
//
|
|
// Get Operating system information from the Fleet instance, we create these records first so we can reference them when saving Host information to the database.
|
|
let allKnownOperatingSystems = await OperatingSystem.find();
|
|
// Build a dictionary containing the last recorded host counts for each Operating system
|
|
let osVersionNamesByHostCount = {};
|
|
for(let osRecord of allKnownOperatingSystems){
|
|
osVersionNamesByHostCount[`${osRecord.fullName}`] = osRecord.lastReportedHostCount;
|
|
}
|
|
// Send a request to the Fleet API to get the current os versions.
|
|
let osVersionsResponseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/os_versions',
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}]);
|
|
|
|
assert(undefined === osVersionsResponseData.os_versions || _.isArray(osVersionsResponseData.os_versions));
|
|
|
|
let currentOSVersions = osVersionsResponseData.os_versions;
|
|
let osVersionsToReport = [];
|
|
let osVersionsToUpdate = [];
|
|
|
|
for(let os of currentOSVersions) {
|
|
if(!os.version) {
|
|
// If an operating system returned in the /os_versions API response is missing a version, we'll log a warning, but we'll still create a record for it with 'N/A' set as the version.
|
|
// This is so we are still able to create host records for hosts with this operating system installed. (The ID of an operating system record is a required value for host records)
|
|
loggedWarningsFromThisScriptRun.push(`An operating system (name: ${os.name}) returned in the response from the /os_versions endpoint is missing a 'version'. This operating system will be reported as having "N/A" as the version. Operating system without a version:`, os);
|
|
os.version = 'N/A';// Note: This does not affect how we match hosts to operating system records.
|
|
}
|
|
let osToReport = {
|
|
name: os.name_only,
|
|
fullName: os.name,
|
|
platform: os.platform,
|
|
versionName: os.version,
|
|
lastReportedHostCount: os.hosts_count
|
|
};
|
|
|
|
if(osVersionNamesByHostCount[os.name] === undefined) {
|
|
osVersionsToReport.push(osToReport);
|
|
} else if(osVersionNamesByHostCount[os.name] !== os.hosts_count) {
|
|
osVersionsToUpdate.push(osToReport);
|
|
}
|
|
}//∞
|
|
|
|
|
|
if(dry){
|
|
sails.log.warn(`Dry run: ${osVersionsToReport.length} new operating system(s), changes to ${osVersionsToUpdate.length} existing operating system(s) detected.`);
|
|
} else {
|
|
sails.log(`${osVersionsToReport.length} new operating system(s), changes to ${osVersionsToUpdate.length} existing operating system(s) detected. saving....`);
|
|
await OperatingSystem.createEach(osVersionsToReport)
|
|
.intercept({name:'UsageError'}, (error)=>{
|
|
return new Error(`When creating new OperatingSystem records from operating systems returned from the Fleet API, an error occured. Full Error: ${error}`);
|
|
});
|
|
for(let osRecordToUpdate of osVersionsToUpdate){
|
|
await OperatingSystem.updateOne({fullName: osRecordToUpdate.fullName}).set({lastReportedHostCount: osRecordToUpdate.lastReportedHostCount});
|
|
}
|
|
}
|
|
|
|
let allOsRecords = await OperatingSystem.find();
|
|
|
|
// ██╗ ██╗ ██████╗ ███████╗████████╗███████╗
|
|
// ██║ ██║██╔═══██╗██╔════╝╚══██╔══╝██╔════╝
|
|
// ███████║██║ ██║███████╗ ██║ ███████╗
|
|
// ██╔══██║██║ ██║╚════██║ ██║ ╚════██║
|
|
// ██║ ██║╚██████╔╝███████║ ██║ ███████║
|
|
// ╚═╝ ╚═╝ ╚═════╝ ╚══════╝ ╚═╝ ╚══════╝
|
|
|
|
// Goal: Collect all hosts on the Fleet instance.
|
|
|
|
let hosts = [];
|
|
{
|
|
let hostsPage = 0;
|
|
await sails.helpers.flow.until( async()=>{
|
|
// sails.log(`sending request for page ${hostsPage} for ${ware.name}.`)
|
|
let responseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/hosts',
|
|
data: { page: hostsPage, per_page: HOSTS_PAGE_SIZE, order_key: 'hostname', order_direction: 'asc' },//eslint-disable-line camelcase
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}])
|
|
.intercept({raw:{statusCode: 404}} , (error)=>{
|
|
return new Error(`When sending a request to the '/api/v1/fleet/hosts' API endpoint to get information about the hosts on this instance, the Fleet instance returned a 404 response when we expected it to return an array of ${HOSTS_PAGE_SIZE} host(s).\n Response from Fleet instance: ${error.raw.body}`);
|
|
});
|
|
// sails.log.warn(`got a response for page ${hostsPage} for ${ware.name}.`);
|
|
if (!responseData.hosts) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return true;
|
|
}//•
|
|
assert(_.isArray(responseData.hosts));
|
|
assert(responseData.hosts.every((host) => _.isNumber(host.id)));
|
|
hosts = hosts.concat(responseData.hosts);
|
|
// console.timeEnd(`Page ${page} of vulnerable software from Fleet API`);
|
|
hostsPage++;
|
|
if(responseData.hosts.length !== HOSTS_PAGE_SIZE){
|
|
return true;
|
|
}
|
|
});
|
|
}//∫
|
|
for (let host of hosts) {
|
|
let hostsOperatingSystem = _.find(allOsRecords, {'fullName': host.os_version});
|
|
if(!hostsOperatingSystem) {
|
|
// Note: In the response form the Hosts API, windows hosts have their os_name prefixed with "Microsoft",
|
|
// So we'll look for an Operating system that matches that format, and log a warning and skip this host if one is not found.
|
|
hostsOperatingSystem = _.find(allOsRecords, {'fullName': 'Microsoft '+host.os_version});
|
|
if(!hostsOperatingSystem){
|
|
sails.log.verbose(`When building host records from the Fleet API, a host's (FleetApid: ${host.id}) os_version (os_version: ${host.os_version}) could not be matched to an operating system returned in the API response from the /os_versions endpoint. Host with missing operating system:`, host);
|
|
continue;
|
|
}
|
|
}
|
|
byHostFleetApidsSeenInLatestFleetScan[host.id] = {
|
|
displayName: _.trim(host.display_name.replace(/,/g, '')),// Trim leading and trailing whitespace, and remove commas from Host names.
|
|
teamDisplayName: host.team_name !== null ? host.team_name : 'No team',
|
|
teamApid: host.team_id !== null ? host.team_id : 0, // Note: If the host is not a member of any team, we'll set this to 0
|
|
fleetApid: host.id,
|
|
hardwareSerialNumber: host.hardware_serial !== '' ? host.hardware_serial : 'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
|
|
uuid: host.uuid,
|
|
operatingSystem: hostsOperatingSystem.id,
|
|
};//∞
|
|
}
|
|
let hostRecordsToUpdate = [];
|
|
// Unrecognized hosts? Save 'em to the database.
|
|
let newRecordsForUnrecognizedHosts = []; {
|
|
let recognizedHosts = await Host.find({ fleetApid: { in: Object.keys(byHostFleetApidsSeenInLatestFleetScan).map((key)=>Number(key)) } });
|
|
let unrecognizedHostApids = _.difference(Object.keys(byHostFleetApidsSeenInLatestFleetScan).map((key)=>Number(key)), _.pluck(recognizedHosts, 'fleetApid'));
|
|
assert(unrecognizedHostApids.every((apid) => _.isNumber(apid)));
|
|
for (let apid of unrecognizedHostApids) {
|
|
newRecordsForUnrecognizedHosts.push(byHostFleetApidsSeenInLatestFleetScan[apid]);
|
|
}//∞
|
|
for(let host of recognizedHosts) {
|
|
let potentialUpdatesToHostRecord = byHostFleetApidsSeenInLatestFleetScan[host.fleetApid];
|
|
// If the host's displayName, teamApid, teamDisplayName, or operating system has changed, we'll update the host record with the latest information.
|
|
if(potentialUpdatesToHostRecord.teamApid !== host.teamApid
|
|
|| potentialUpdatesToHostRecord.teamDisplayName !== host.teamDisplayName
|
|
|| potentialUpdatesToHostRecord.operatingSystem !== host.operatingSystem
|
|
|| potentialUpdatesToHostRecord.displayName !== host.displayName)
|
|
{
|
|
// Add the database id of this host to the record to update, and add it to the array of hostRecordsToUpdate.
|
|
potentialUpdatesToHostRecord.id = host.id;
|
|
hostRecordsToUpdate.push(potentialUpdatesToHostRecord);
|
|
}
|
|
}
|
|
}//∫
|
|
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${newRecordsForUnrecognizedHosts.length} hosts were seemingly enrolled. (Fleet returned them in the API.)`);
|
|
sails.log.warn(`Dry run: ${hostRecordsToUpdate.length} hosts will be updated with new information. (Fleet returned them in the API.)`);
|
|
} else {
|
|
sails.log(`Creating ${newRecordsForUnrecognizedHosts.length} host records… `);
|
|
let batchedNewRecordsForUnrecognizedHosts = _.chunk(newRecordsForUnrecognizedHosts, 500);
|
|
for(let batch of batchedNewRecordsForUnrecognizedHosts){
|
|
await Host.createEach(batch);
|
|
}
|
|
for(let hostUpdate of hostRecordsToUpdate){
|
|
await Host.updateOne({id: hostUpdate.id}).set(_.omit(hostUpdate, 'id'));
|
|
}
|
|
}
|
|
|
|
// ██╗ ██╗██╗ ██╗██╗ ███╗ ██╗███████╗██████╗ █████╗ ██████╗ ██╗ ███████╗
|
|
// ██║ ██║██║ ██║██║ ████╗ ██║██╔════╝██╔══██╗██╔══██╗██╔══██╗██║ ██╔════╝
|
|
// ██║ ██║██║ ██║██║ ██╔██╗ ██║█████╗ ██████╔╝███████║██████╔╝██║ █████╗
|
|
// ╚██╗ ██╔╝██║ ██║██║ ██║╚██╗██║██╔══╝ ██╔══██╗██╔══██║██╔══██╗██║ ██╔══╝
|
|
// ╚████╔╝ ╚██████╔╝███████╗██║ ╚████║███████╗██║ ██║██║ ██║██████╔╝███████╗███████╗
|
|
// ╚═══╝ ╚═════╝ ╚══════╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═════╝ ╚══════╝╚══════╝
|
|
//
|
|
// ███████╗ ██████╗ ███████╗████████╗██╗ ██╗ █████╗ ██████╗ ███████╗
|
|
// ██╔════╝██╔═══██╗██╔════╝╚══██╔══╝██║ ██║██╔══██╗██╔══██╗██╔════╝
|
|
// ███████╗██║ ██║█████╗ ██║ ██║ █╗ ██║███████║██████╔╝█████╗
|
|
// ╚════██║██║ ██║██╔══╝ ██║ ██║███╗██║██╔══██║██╔══██╗██╔══╝
|
|
// ███████║╚██████╔╝██║ ██║ ╚███╔███╔╝██║ ██║██║ ██║███████╗
|
|
// ╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚══╝╚══╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝
|
|
//
|
|
await sails.helpers.flow.until(async ()=>{
|
|
// * * *
|
|
// Load a page of vulnerable software versions.
|
|
// [?] https://fleetdm.com/docs/using-fleet/rest-api#list-all-software
|
|
// # requests == O(vulnerableSoftware / SOFTWARE_VERSIONS_PAGE_SIZE)
|
|
let vulnerableWares;
|
|
{
|
|
let responseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/software',
|
|
data: { vulnerable: true, page: page, per_page: SOFTWARE_VERSIONS_PAGE_SIZE, order_key: 'name', order_direction: 'asc' },//eslint-disable-line camelcase
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}]); //
|
|
assert(undefined === responseData.software || _.isArray(responseData.software));
|
|
if (!responseData.software) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return true;
|
|
}//•
|
|
vulnerableWares = responseData.software;
|
|
}//∫
|
|
|
|
sails.log.info('Processing page #',page,'of Fleet vulnerability data (',vulnerableWares.length,'software items )');// sails.log.verbose(require('util').inspect(vulnerableWares,{depth:null}));
|
|
// * * *
|
|
// For each software version, look up affected hosts.
|
|
// (i.e. they have this version of software installed)
|
|
let hostApidsBySoftwareVersionApid = {};// « Save a mapping for use below.
|
|
let vulnerableWaresWithNoHostInformation = [];
|
|
await sails.helpers.flow.simultaneouslyForEach(vulnerableWares, async(ware)=>{
|
|
// Get hosts with this version of software installed.
|
|
// [?] https://fleetdm.com/docs/using-fleet/rest-api#list-hosts
|
|
// # requests == O(SOFTWARE_VERSIONS_PAGE_SIZE * (vulnerableSoftware / SOFTWARE_VERSIONS_PAGE_SIZE))
|
|
let hosts = [];
|
|
{
|
|
let hostsPage = 0;
|
|
await sails.helpers.flow.until( async()=>{
|
|
let responseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/hosts',
|
|
data: { software_id: ware.id, page: hostsPage, per_page: HOSTS_PAGE_SIZE, order_key: 'hostname', order_direction: 'asc' },//eslint-disable-line camelcase
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}])
|
|
.tolerate({raw:{statusCode: 404}} , ()=>{
|
|
// If the hosts API returns a 404 response for a software item that was returned from in the list of vulnerable software, we'll log a warning and remove this software from the list of software.
|
|
loggedWarningsFromThisScriptRun.push(`When processing vulnerable software, a request to the '/hosts' endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Impact: If this vulnerable software was previously processed, the database record(s) for it will be marked as uninstalled. If it shows up in a future run of this script, a new database record will be created.`);
|
|
vulnerableWaresWithNoHostInformation.push(ware);// Add this software to the vulnerableWaresWithNoHostInformation array, these will be removed before we create and update database records.
|
|
return {};// Return an empty object. This will let the script continue without information about this software.
|
|
});
|
|
if (!responseData.hosts) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return true;
|
|
}//•
|
|
assert(_.isArray(responseData.hosts));
|
|
assert(responseData.hosts.every((host) => _.isNumber(host.id)));
|
|
hosts = hosts.concat(responseData.hosts);
|
|
hostsPage++;
|
|
if(responseData.hosts.length !== HOSTS_PAGE_SIZE){
|
|
return true;
|
|
}
|
|
});
|
|
}//∫
|
|
|
|
|
|
hostApidsBySoftwareVersionApid[ware.id] = _.pluck(hosts, 'id');// Keep track of which hosts have this software installed, for use below w/ vulns.
|
|
for (let host of hosts) {
|
|
let hostsOperatingSystem = _.find(allOsRecords, {'fullName': host.os_version});
|
|
if(!hostsOperatingSystem) {
|
|
// Note: In the response form the Hosts API, windows hosts have their os_name prefixed with "Microsoft",
|
|
// So we'll look for an Operating system that matches that format, and log a warning and skip this host if one is not found.
|
|
hostsOperatingSystem = _.find(allOsRecords, {'fullName': 'Microsoft '+host.os_version});
|
|
if(!hostsOperatingSystem){
|
|
sails.log.verbose(`When building host records from the Fleet API, a host's (FleetApid: ${host.id}) os_version (os_version: ${host.os_version}) could not be matched to an operating system returned in the API response from the /os_versions endpoint. Host with missing operating system:`, host);
|
|
continue;
|
|
}
|
|
}
|
|
byHostFleetApidsSeenInLatestFleetScan[host.id] = {
|
|
displayName: _.trim(host.display_name.replace(/,/g, '')),// Trim leading and trailing whitespace, and remove commas from Host names.
|
|
teamDisplayName: host.team_name !== null ? host.team_name : 'No team',
|
|
teamApid: host.team_id !== null ? host.team_id : 0, // Note: If the host is not a member of any team, we'll set this to 0
|
|
fleetApid: host.id,
|
|
hardwareSerialNumber: host.hardware_serial !== '' ? host.hardware_serial : 'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
|
|
uuid: host.uuid,
|
|
operatingSystem: hostsOperatingSystem.id,
|
|
};//∞
|
|
}
|
|
}).intercept((err)=>{
|
|
sails.log.error('An error occurred in side the iteratee of a flow control function. Due to an unresolved issue, these errors can be hard to dissect without code changes. Just in case, this log message seeks to dump as much info as possible!\n','err',err,'\nerr.raw', err.raw);
|
|
// // Since .raw isn't displayed in output again in Node ≥12, try to make the error better.
|
|
if (err.code === 'E_INTERNAL_ERROR') {
|
|
return err.raw;
|
|
} else {
|
|
return err;
|
|
}
|
|
});//∞ </each software version>
|
|
|
|
// Remove any software items that was not returned in the hosts API.
|
|
vulnerableWares = _.difference(vulnerableWares, vulnerableWaresWithNoHostInformation);
|
|
|
|
let hostRecordsToUpdate = [];
|
|
// Unrecognized hosts? Save 'em to the database.
|
|
let newRecordsForUnrecognizedHosts = []; {
|
|
let recognizedHosts = await Host.find({ fleetApid: { in: Object.keys(byHostFleetApidsSeenInLatestFleetScan).map((key)=>Number(key)) } });
|
|
let unrecognizedHostApids = _.difference(Object.keys(byHostFleetApidsSeenInLatestFleetScan).map((key)=>Number(key)), _.pluck(recognizedHosts, 'fleetApid'));
|
|
assert(unrecognizedHostApids.every((apid) => _.isNumber(apid)));
|
|
for (let apid of unrecognizedHostApids) {
|
|
newRecordsForUnrecognizedHosts.push(byHostFleetApidsSeenInLatestFleetScan[apid]);
|
|
}//∞
|
|
for(let host of recognizedHosts) {
|
|
let potentialUpdatesToHostRecord = byHostFleetApidsSeenInLatestFleetScan[host.fleetApid];
|
|
// If the host's displayName, teamApid, teamDisplayName, or operating system has changed, we'll update the host record with the latest information.
|
|
if(potentialUpdatesToHostRecord.teamApid !== host.teamApid
|
|
|| potentialUpdatesToHostRecord.teamDisplayName !== host.teamDisplayName
|
|
|| potentialUpdatesToHostRecord.operatingSystem !== host.operatingSystem
|
|
|| potentialUpdatesToHostRecord.displayName !== host.displayName)
|
|
{
|
|
// Add the database id of this host to the record to update, and add it to the array of hostRecordsToUpdate.
|
|
potentialUpdatesToHostRecord.id = host.id;
|
|
hostRecordsToUpdate.push(potentialUpdatesToHostRecord);
|
|
|
|
}
|
|
|
|
}
|
|
}//∫
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${newRecordsForUnrecognizedHosts.length} hosts were seemingly enrolled. (Fleet returned them in the API.)`);
|
|
sails.log.warn(`Dry run: ${hostRecordsToUpdate.length} hosts will be updated with new information. (Fleet returned them in the API.)`);
|
|
} else {
|
|
sails.log(`Creating ${newRecordsForUnrecognizedHosts.length} host records… `);
|
|
let batchedNewRecordsForUnrecognizedHosts = _.chunk(newRecordsForUnrecognizedHosts, 500);
|
|
for(let batch of batchedNewRecordsForUnrecognizedHosts){
|
|
await Host.createEach(batch);
|
|
}
|
|
for(let hostUpdate of hostRecordsToUpdate){
|
|
await Host.updateOne({id: hostUpdate.id}).set(_.omit(hostUpdate, 'id'));
|
|
}
|
|
}
|
|
|
|
// * * *
|
|
// Build a formatted, filtered set of the vulnerabilities returned by this scan.
|
|
// These will be analyzed below and potentially written to the database.
|
|
let potentiallyNewVulnRecords = [];
|
|
let cveIdsBySoftwareVersionApid = {};// « for use below when creating vuln installation records
|
|
for (let ware of vulnerableWares) {
|
|
softwareInfoForVulnInstalls[ware.id] = {
|
|
softwareName: ware.name.replace(/,/g, ''),// Remove any commas in software names
|
|
versionName: ware.version !== '' ? ware.version.replace(/,/g, '') : 'N/A',// Remove any commas in version names. if a version name is missing, this will be added as 'N/A'
|
|
resolvedVersionsByCveId: {},
|
|
};
|
|
for(let vuln of ware.vulnerabilities){
|
|
softwareInfoForVulnInstalls[ware.id].resolvedVersionsByCveId[vuln.cve] = vuln.resolved_in_version !== null ? vuln.resolved_in_version : '';
|
|
}
|
|
cveIdsBySoftwareVersionApid[ware.id] = [];
|
|
for (let fleetVuln of ware.vulnerabilities) {
|
|
assert(_.isObject(fleetVuln));
|
|
cveIdsBySoftwareVersionApid[ware.id].push(fleetVuln.cve);
|
|
if (!fleetVuln.cve_published) {
|
|
// sails.log.warn('Unrecognized CVE detected in Fleet:' + require('util').inspect(fleetVuln, {depth:null}), 'for a software with the ID of '+ware.id+'. This means that the data being used to hydrate the vulnerability data with additional info such as publish date is missing this particular CVE, even though it was reported in the results from scanning with Fleet. For consistency, excluding this vulnerability from the Fleet scan results as if it was not detected.');
|
|
} else if (!fleetVuln.cvss_score) {
|
|
// sails.log.verbose('Invalid CVSS score detected by Fleet:' + require('util').inspect(fleetVuln, {depth:null}), 'This might mean that the vulnerability is undergoing reanlysis. For now, this vulnerability will be treated as if it was not detected in the Fleet scan results.');
|
|
} else {
|
|
if (!_.any(potentiallyNewVulnRecords, {cveId: fleetVuln.cve})) {// Don't track duplicates
|
|
potentiallyNewVulnRecords.push({
|
|
cveId: fleetVuln.cve,
|
|
fleetSoftwareItemUrl: `${fleetBaseUrl}/software/${encodeURIComponent(ware.id)}`,
|
|
additionalDetailsUrl: fleetVuln.details_link,
|
|
probabilityOfExploit: fleetVuln.epss_probability !== null ? fleetVuln.epss_probability : 0,// If the Fleet server sends this value as null, we'll set this to be 0.
|
|
severity: fleetVuln.cvss_score,
|
|
cveDescription: fleetVuln.cve_description.replace(/,/g, ''),
|
|
hasKnownExploit: fleetVuln.cisa_known_exploit,
|
|
publishedAt: new Date(fleetVuln.cve_published).getTime(),
|
|
});
|
|
}
|
|
}
|
|
}//∞
|
|
}//∞
|
|
// * * *
|
|
// Determine any new vulns that need to be created, then create records for them!
|
|
// let $knownVulns = _.filter(allKnownVulns, (vuln)=>{return _.contains(_.pluck(potentiallyNewVulnRecords, 'cveId'), vuln.cveId)});
|
|
let $knownVulns = await Vulnerability.find({
|
|
cveId: { in: _.pluck(potentiallyNewVulnRecords, 'cveId') }
|
|
});
|
|
let newlyDiscoveredCveIds = _.difference(_.pluck(potentiallyNewVulnRecords, 'cveId'), _.pluck($knownVulns, 'cveId'));
|
|
let previouslyDiscoveredVulns = _.intersection(_.pluck(potentiallyNewVulnRecords, 'cveId'), _.pluck($knownVulns, 'cveId'));
|
|
let newlyDiscoveredVulns = [];
|
|
let newlyUpdatedVulns = [];
|
|
for (let cveId of newlyDiscoveredCveIds) {
|
|
newlyDiscoveredVulns.push(_.find(potentiallyNewVulnRecords, {cveId: cveId}));
|
|
}//∞
|
|
for (let cveId of previouslyDiscoveredVulns) {
|
|
// check to see if this vulnerabilities information has changed.
|
|
if(_.find(potentiallyNewVulnRecords, {cveId: cveId}).cveDescription !== _.find($knownVulns, {cveId: cveId}).cveDescription){
|
|
newlyUpdatedVulns.push(_.find(potentiallyNewVulnRecords, {cveId: cveId}));
|
|
}
|
|
}//∞
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${newlyDiscoveredCveIds.length} newly discovered vulnerabilities (CVEs) are available and updates to ${newlyUpdatedVulns.length} vulnerabilities were found. (Fleet returned them in the API.)`);
|
|
} else {
|
|
sails.log(`Detected ${newlyDiscoveredCveIds.length} new vulnerabilities and updates to ${newlyUpdatedVulns.length} vulnerabilities were found.. Saving...`);
|
|
await Vulnerability.createEach(newlyDiscoveredVulns).fetch();
|
|
for(let vuln of newlyUpdatedVulns){
|
|
await Vulnerability.updateOne({cveId: vuln.cveId}).set(vuln);
|
|
}
|
|
}
|
|
|
|
// * * *
|
|
// Build a set of vuln installation records to be saved.
|
|
let potentialVulnInstalls = [];
|
|
let potentialVulnInstallUpdates = [];
|
|
|
|
for (let ware of vulnerableWares) {
|
|
let $vulns = await Vulnerability.find({ cveId: { in: cveIdsBySoftwareVersionApid[ware.id] }});// FUTURE: optimize (O(n) queries)
|
|
let $hosts = await Host.find({ fleetApid: { in: hostApidsBySoftwareVersionApid[ware.id] }});// FUTURE: optimize (O(n) queries)
|
|
// console.log(ware.id+': '+$vulns.length +' vulnerabilities, '+$hosts.length +' hosts');
|
|
for (let $vuln of $vulns) {
|
|
for (let $host of $hosts) {
|
|
// If a VulnerabilityInstall record exists that has the same host and vulnerability as the one we detected, we'll ignore it.
|
|
let vulnInstallExists = existingVulnInstallsByHostAndVulnIDs[`${ware.id}|${$vuln.id}|${$host.id}`];
|
|
|
|
// Set the missingVulnInstallsByIds value for this software/host/vulnerability combination to be false.
|
|
missingVulnInstallsByIds[`${ware.id}|${$vuln.id}|${$host.id}`] = false;
|
|
let thisVulnInstall = {
|
|
installedAt: Date.now(),
|
|
host: $host.id,
|
|
vulnerability: $vuln.id,
|
|
fleetApid: Number(ware.id),
|
|
softwareName: softwareInfoForVulnInstalls[ware.id].softwareName,
|
|
versionName: softwareInfoForVulnInstalls[ware.id].versionName,
|
|
resolvedInVersion: softwareInfoForVulnInstalls[ware.id].resolvedVersionsByCveId[$vuln.cveId],
|
|
};
|
|
if(!vulnInstallExists){
|
|
potentialVulnInstalls.push(thisVulnInstall);
|
|
// Add this install to the existingVulnInstallsByHostAndVulnIDs dictionary
|
|
existingVulnInstallsByHostAndVulnIDs[`${ware.id}|${$vuln.id}|${$host.id}`] = true;
|
|
} else if(thisVulnInstall.resolvedInVersion) {// If this vulnerable software has a resolvedInVersion value, we'll check to see if it is what we previosly recorded.
|
|
// If a record for this vuln install exists, we'll check the affectedVersionName value of the existing record to see if it should be updated.
|
|
if(vulnInstallsResolvedVersionsByApids[`${ware.id}|${$vuln.id}`] !== thisVulnInstall.resolvedInVersion){
|
|
potentialVulnInstallUpdates.push(thisVulnInstall);
|
|
}
|
|
}
|
|
}//∞
|
|
}//∞
|
|
}//∞
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${potentialVulnInstalls.length} potential vulnerability installs are available and ${potentialVulnInstallUpdates.length} updates are available for existing vulnerability installs.`);
|
|
} else {
|
|
sails.log(`Detected ${potentialVulnInstalls.length} changes to software installations and ${potentialVulnInstallUpdates.length} updates are available for existing vulnerability installs. Saving...`);
|
|
let batchedPotentialVulnInstalls = _.chunk(potentialVulnInstalls, 1000);
|
|
for(let batch of batchedPotentialVulnInstalls){
|
|
await VulnerabilityInstall.createEach(batch);
|
|
}
|
|
for(let install of potentialVulnInstallUpdates){
|
|
await VulnerabilityInstall.update({fleetApid: install.fleetApid, vulnerability: install.vulnerability})
|
|
.set({
|
|
resolvedInVersion: install.resolvedInVersion
|
|
});
|
|
}
|
|
}
|
|
for (let vuln of potentiallyNewVulnRecords) {
|
|
byCveIdsSeenInLatestFleetScan[vuln.cveId] = true;
|
|
}//∞
|
|
numVulnerableWaresProcessed += vulnerableWares.length;
|
|
page++;
|
|
}, 90 * 60 * 1000)// (timeout after 90 mintutes)
|
|
.intercept((err) =>
|
|
require('flaverr')({message: 'Could not get software from the Fleet API. Error details: '+err.message}, err)
|
|
);
|
|
// ██████╗██████╗ ██╗████████╗██╗ ██████╗ █████╗ ██╗
|
|
// ██╔════╝██╔══██╗██║╚══██╔══╝██║██╔════╝██╔══██╗██║
|
|
// ██║ ██████╔╝██║ ██║ ██║██║ ███████║██║
|
|
// ██║ ██╔══██╗██║ ██║ ██║██║ ██╔══██║██║
|
|
// ╚██████╗██║ ██║██║ ██║ ██║╚██████╗██║ ██║███████╗
|
|
// ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚══════╝
|
|
//
|
|
// ███████╗ ██████╗ ███████╗████████╗██╗ ██╗ █████╗ ██████╗ ███████╗
|
|
// ██╔════╝██╔═══██╗██╔════╝╚══██╔══╝██║ ██║██╔══██╗██╔══██╗██╔════╝
|
|
// ███████╗██║ ██║█████╗ ██║ ██║ █╗ ██║███████║██████╔╝█████╗
|
|
// ╚════██║██║ ██║██╔══╝ ██║ ██║███╗██║██╔══██║██╔══██╗██╔══╝
|
|
// ███████║╚██████╔╝██║ ██║ ╚███╔███╔╝██║ ██║██║ ██║███████╗
|
|
// ╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚══╝╚══╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝
|
|
//
|
|
let QUERIES_TO_GET_CRITICAL_SOFTWARE = [
|
|
{query: 'Safari.app', type: 'safari'},// Safari on macOS
|
|
{query: 'Firefox.app', type: 'firefox'},// Safari on macOS
|
|
{query: 'Google Chrome', type: 'chrome'},// Safari on macOS
|
|
{query: 'Google Chrome.app', type: 'chrome'},// Safari on macOS
|
|
{query: 'Firefox', type: 'firefox'},// Firefox on Linux
|
|
{query: 'Mozilla Firefox', type: 'firefox'},// Firefox on Windows?
|
|
{query: 'Flash player.app', type: 'flash'}, // Flash on macOS
|
|
{query: 'Flash', type: 'flash'},// Flash on Windows.
|
|
{query: 'Microsoft Excel', type: 'microsoftOffice'},// Microsoft Excel on all platforms.
|
|
{query: 'Microsoft Powerpoint', type: 'microsoftOffice'},// Microsoft Powerpoint on all platforms.
|
|
{query: 'Microsoft Word', type: 'microsoftOffice'},// Microsoft Word on all platforms.
|
|
{query: 'Microsoft Outlook', type: 'microsoftOffice'},// Microsoft Outlook on all platforms.
|
|
];
|
|
let totalNumberOfHostRecordsCreated = 0;
|
|
let totalNumberOfHostRecordsUpdated = 0;
|
|
let potentialCriticalInstalls = [];
|
|
let criticalWaresWithNoHostInformation = [];
|
|
await sails.helpers.flow.forEach(QUERIES_TO_GET_CRITICAL_SOFTWARE, async (softwareQuery)=>{
|
|
// Send a request to the software endpoint for each query to get a list of software isntalled for each query.
|
|
let criticalWares;
|
|
{
|
|
let responseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/software',
|
|
data: { query: softwareQuery.query },
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}]); //
|
|
assert(undefined === responseData.software || _.isArray(responseData.software));
|
|
if (!responseData.software) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return;
|
|
}//•
|
|
// Filter the results based on the type of software we're looking for. (Note: we filter browser extensions for all critical software)
|
|
if(softwareQuery.type === 'flash') {
|
|
criticalWares = responseData.software.filter((software)=>{
|
|
// For Flash, we'll only track software items if the name matches exactly.
|
|
if(software.source !== 'firefox_addons' && software.source !== 'chrome_extensions'){
|
|
return software.name.toLowerCase() === softwareQuery.query.toLowerCase();
|
|
}
|
|
});
|
|
} else if(softwareQuery.type === 'chrome') {
|
|
criticalWares = responseData.software.filter((software)=>{
|
|
// if we're looking for chrome installs, we'll filter out results with the word "Helper" in it
|
|
if(software.source !== 'firefox_addons' && software.source !== 'chrome_extensions' && !software.name.match(/helper/gi)) {
|
|
return _.startsWith(software.name.toLowerCase(), softwareQuery.query.toLowerCase());
|
|
}
|
|
});
|
|
} else {
|
|
// For any other software tyoe, we'll just filter out browser extensions.
|
|
criticalWares = responseData.software.filter((software)=>{
|
|
if(software.source !== 'firefox_addons' && software.source !== 'chrome_extensions'){
|
|
return _.startsWith(software.name.toLowerCase(), softwareQuery.query.toLowerCase());
|
|
}
|
|
});
|
|
}
|
|
}//∫
|
|
let hostApidsBySoftwareVersionApid = {};
|
|
await sails.helpers.flow.simultaneouslyForEach(criticalWares, async(ware)=>{
|
|
// Get hosts with this version of software installed.
|
|
let hosts = [];
|
|
{
|
|
let hostsPage = 0;
|
|
await sails.helpers.flow.until( async()=>{
|
|
// sails.log(`sending request for page ${hostsPage} for ${ware.name}.`)
|
|
let responseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/hosts',
|
|
data: { software_id: ware.id, page: hostsPage, per_page: HOSTS_PAGE_SIZE, order_key: 'hostname', order_direction: 'asc' },//eslint-disable-line camelcase
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}])
|
|
.tolerate({raw:{statusCode: 404}} , ()=>{
|
|
// If the hosts API returns a 404 response for a software item that was returned from in the list of critical software, we'll log a warning and remove this software from the list of software.
|
|
loggedWarningsFromThisScriptRun.push(`When processing critical software, a request to the '/hosts' endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Impact: This software will be marked as uninstalled, and a new database record will be created if it shows up in a future run of this script.`);
|
|
criticalWaresWithNoHostInformation.push(ware);// Add this software to the criticalWaresWithNoHostInformation array, these will be removed before we create and update database records.
|
|
return {};// Return an empty object. This will let the script continue without information about this software.
|
|
});
|
|
// sails.log.warn(`got a response for page ${hostsPage} for ${ware.name}.`);
|
|
if (!responseData.hosts) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return true;
|
|
}//•
|
|
assert(_.isArray(responseData.hosts));
|
|
assert(responseData.hosts.every((host) => _.isNumber(host.id)));
|
|
hosts = hosts.concat(responseData.hosts);
|
|
hostsPage++;
|
|
if(responseData.hosts.length !== HOSTS_PAGE_SIZE){
|
|
return true;
|
|
}
|
|
});
|
|
}//∫
|
|
hostApidsBySoftwareVersionApid[ware.id] = _.pluck(hosts, 'id');// Keep track of which hosts have this software installed, for use below w/ vulns.
|
|
for (let host of hosts) {
|
|
let hostsOperatingSystem = _.find(allOsRecords, {'fullName': host.os_version});
|
|
if(!hostsOperatingSystem){
|
|
// Note: In the response form the Hosts API, windows hosts have their os_name prefixed with "Microsoft",
|
|
// So we'll look for an Operating system that matches that format, and log a warning and skip this host if one is not found.
|
|
hostsOperatingSystem = _.find(allOsRecords, {'fullName': 'Microsoft '+host.os_version});
|
|
if(!hostsOperatingSystem){
|
|
sails.log.verbose(`When building host records from the Fleet API, a host's (FleetApid: ${host.id}) os_version (os_version: ${host.os_version}) could not be matched to an operating system returned in the API response from the /os_versions endpoint. Host with missing operating system:`, host);
|
|
continue;
|
|
}
|
|
}
|
|
byHostFleetApidsSeenInLatestCriticalSoftwareScan[host.id] = {
|
|
displayName: _.trim(host.display_name.replace(/,/g, '')),// Trim leading and trailing whitespace, and remove commas from Host names.
|
|
teamDisplayName: host.team_name !== null ? host.team_name : 'No team',
|
|
teamApid: host.team_id !== null ? host.team_id : 0, // Note: If the host is not a member of any team, we'll set this to 0
|
|
fleetApid: host.id,
|
|
hardwareSerialNumber: host.hardware_serial !== '' ? host.hardware_serial : 'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
|
|
uuid: host.uuid,
|
|
operatingSystem: hostsOperatingSystem.id,
|
|
};//∞
|
|
}
|
|
}).intercept((err)=>{
|
|
sails.log.error('An error occurred in side the iteratee of a flow control function. Due to an unresolved issue, these errors can be hard to dissect without code changes. Just in case, this log message seeks to dump as much info as possible!\n','err',err,'\nerr.raw', err.raw);
|
|
// // Since .raw isn't displayed in output again in Node ≥12, try to make the error better.
|
|
if (err.code === 'E_INTERNAL_ERROR') {
|
|
return err.raw;
|
|
} else {
|
|
return err;
|
|
}
|
|
});//∞ </each software version>
|
|
|
|
// Remove any software items that was not returned in the hosts API.
|
|
criticalWares = _.difference(criticalWares, criticalWaresWithNoHostInformation);
|
|
|
|
let hostRecordsToUpdate = [];
|
|
// Unrecognized hosts? Save 'em to the database.
|
|
let newRecordsForUnrecognizedHosts = [];
|
|
{
|
|
let recognizedHosts = await Host.find({ fleetApid: { in: Object.keys(byHostFleetApidsSeenInLatestCriticalSoftwareScan).map((key)=>Number(key)) } });
|
|
let unrecognizedHostApids = _.difference(Object.keys(byHostFleetApidsSeenInLatestCriticalSoftwareScan).map((key)=>Number(key)), _.pluck(recognizedHosts, 'fleetApid'));
|
|
assert(unrecognizedHostApids.every((apid) => _.isNumber(apid)));
|
|
for (let apid of unrecognizedHostApids) {
|
|
newRecordsForUnrecognizedHosts.push(byHostFleetApidsSeenInLatestCriticalSoftwareScan[apid]);
|
|
}//∞
|
|
for(let host of recognizedHosts) {
|
|
let potentialUpdatesToHostRecord = byHostFleetApidsSeenInLatestCriticalSoftwareScan[host.fleetApid];
|
|
// If the host's displayName, teamApid, teamDisplayName, or operating system has changed, we'll update the host record with the latest information.
|
|
if(potentialUpdatesToHostRecord.teamApid !== host.teamApid
|
|
|| potentialUpdatesToHostRecord.teamDisplayName !== host.teamDisplayName
|
|
|| potentialUpdatesToHostRecord.operatingSystem !== host.operatingSystem
|
|
|| potentialUpdatesToHostRecord.displayName !== host.displayName)
|
|
{
|
|
// Add the database id of this host to the record to update, and add it to the array of hostRecordsToUpdate.
|
|
potentialUpdatesToHostRecord.id = host.id;
|
|
hostRecordsToUpdate.push(potentialUpdatesToHostRecord);
|
|
}
|
|
|
|
}
|
|
}//∫
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${newRecordsForUnrecognizedHosts.length} hosts were seemingly enrolled. (Fleet returned them in the API.)`);
|
|
sails.log.warn(`Dry run: ${hostRecordsToUpdate.length} hosts will be updated with new information. (Fleet returned them in the API.)`);
|
|
} else {
|
|
totalNumberOfHostRecordsCreated += newRecordsForUnrecognizedHosts.length;
|
|
totalNumberOfHostRecordsUpdated += hostRecordsToUpdate.length;
|
|
sails.log.verbose(`Creating ${newRecordsForUnrecognizedHosts.length} new host records…`);
|
|
let batchedNewRecordsForUnrecognizedHosts = _.chunk(newRecordsForUnrecognizedHosts, 500);
|
|
for(let batch of batchedNewRecordsForUnrecognizedHosts){
|
|
await Host.createEach(batch);
|
|
}
|
|
sails.log.verbose(`Updating ${hostRecordsToUpdate.length} host records…`);
|
|
for(let hostUpdate of hostRecordsToUpdate){
|
|
await Host.updateOne({id: hostUpdate.id}).set(_.omit(hostUpdate, 'id'));
|
|
}
|
|
}
|
|
|
|
|
|
for (let ware of criticalWares) {
|
|
let $hosts = await Host.find({ fleetApid: { in: hostApidsBySoftwareVersionApid[ware.id] }});
|
|
for (let $host of $hosts) {
|
|
// If a CriticalInstall record exists that has the same host and vulnerability as the one we detected, we'll ignore it.
|
|
let criticalInstallExists = existingCriticalInstallsByHostIDs[`${ware.id}|${$host.id}`];
|
|
let isCompliant = false;// Default to false.
|
|
if(softwareQuery.type === 'microsoftOffice') {
|
|
// Since Microsoft office is a suite of software that share a version, we'll check to see if this
|
|
// version of Microsoft office has been marked as compliant to determine if this new version is compliant
|
|
isCompliant = _.contains(compliantMicrosoftOfficeVersions, ware.version);
|
|
} else {
|
|
// Otherwise, check to see if this software's API ID has been previosuly marked as compliant.
|
|
isCompliant = _.contains(compliantVersionsApids, ware.id);
|
|
}
|
|
missingCriticalInstallsByHostIDs[`${ware.id}|${$host.id}`] = false;// « ex: {'140|2146':true, '135|6729':true, ...}
|
|
if(!criticalInstallExists) {
|
|
potentialCriticalInstalls.push({
|
|
installedAt: Date.now(),
|
|
host: $host.id,
|
|
fleetApid: Number(ware.id),
|
|
softwareName: ware.name.replace(/.app$/i, ''),
|
|
versionName: ware.version,
|
|
softwareType: softwareQuery.type,
|
|
platform: _.find(allOsRecords, {id: $host.operatingSystem}).platform,
|
|
isCompliant,
|
|
});
|
|
// Add this install to the existingCriticalInstallsByHostIDs dictionary
|
|
existingCriticalInstallsByHostIDs[`${ware.id}|${$host.id}`] = true;
|
|
}
|
|
}//∞
|
|
}//∞
|
|
});
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${potentialCriticalInstalls.length} potential critical installs are available.`);
|
|
} else {
|
|
sails.log(`Created ${totalNumberOfHostRecordsCreated} new host records and updated ${totalNumberOfHostRecordsUpdated} existing host records. `);
|
|
sails.log(`Detected ${potentialCriticalInstalls.length} changes to critical software installations. Saving...`);
|
|
let batchedPotentialCriticalInstalls = _.chunk(potentialCriticalInstalls, 1000);
|
|
for(let batch of batchedPotentialCriticalInstalls){
|
|
await CriticalInstall.createEach(batch);
|
|
}
|
|
}
|
|
|
|
// ┬─┐┌─┐┌─┐┌─┐┌┬┐ ┌┬┐┌─┐ ┬─┐┌─┐┌─┐┌─┐┬ ┬ ┬┌─┐┌┬┐ ┬ ┬┬ ┬┬ ┌┐┌┌─┐
|
|
// ├┬┘├┤ ├─┤│ │ │ │ │ ├┬┘├┤ └─┐│ ││ └┐┌┘├┤ ││ └┐┌┘│ ││ │││└─┐
|
|
// ┴└─└─┘┴ ┴└─┘ ┴ ┴ └─┘ ┴└─└─┘└─┘└─┘┴─┘└┘ └─┘─┴┘ └┘ └─┘┴─┘┘└┘└─┘
|
|
// ┬ ┌┬┐┬┌─┐┌─┐┬┌┐┌┌─┐ ┬ ┬┌─┐┌─┐┌┬┐┌─┐
|
|
// ┌┼─ ││││└─┐└─┐│││││ ┬ ├─┤│ │└─┐ │ └─┐
|
|
// └┘ ┴ ┴┴└─┘└─┘┴┘└┘└─┘ ┴ ┴└─┘└─┘ ┴ └─┘
|
|
// * * *
|
|
// Check for newly-fixed vulnerabilities.
|
|
// (i.e. newly-fixed means previously-detected vulns that are no longer present)
|
|
let newlyFixedVulns = [];
|
|
// > Note: This could be further optimized for huge datasets to update vulnerabilities gradually
|
|
// > as each batch is streamed. But it really only matters when many many vulns are fixed all at
|
|
// > once (i.e. `newlyFixedVulns`` overflowing).
|
|
await Vulnerability.stream()
|
|
.meta({batchSize: 1000})
|
|
.eachRecord(($vuln)=>{
|
|
// if (!Object.keys(byCveIdsSeenInLatestFleetScan).includes($vuln.cveId)) {
|
|
if (!byCveIdsSeenInLatestFleetScan[$vuln.cveId]) {
|
|
newlyFixedVulns.push($vuln);
|
|
}
|
|
});//∞
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${newlyFixedVulns.length} previously-installed vulnerabilities were seemingly fixed. (Fleet did not return them in the API this time.)`);
|
|
} else {
|
|
// TODO: Verify that these are correct
|
|
sails.log(`${newlyFixedVulns.length} previously-installed vulnerabilities were seemingly fixed. (Fleet did not return them in the API this time.)`);
|
|
let batchesOfNewlyFixedVulnInstalls = _.chunk(_.pluck(newlyFixedVulns,'id'), 5000);
|
|
for(let batch of batchesOfNewlyFixedVulnInstalls) {
|
|
await VulnerabilityInstall.update({ vulnerability: { in: batch }, uninstalledAt: 0 }).set({
|
|
uninstalledAt: Date.now()
|
|
});
|
|
}//∞
|
|
}
|
|
|
|
let vulnInstallRecordIdsThatNoLongerExistInFleetInstance = [];// « Create an array to store the database IDs of installs that Fleet did not return in the API response.
|
|
// This check is here to handles cases where vulnerable software is updated on a host, but the new version is affected by the same vulnerability as the old version.
|
|
for(let installToCheck in missingVulnInstallsByIds) {
|
|
if(missingVulnInstallsByIds[installToCheck]) {// This will be set to false if this install was included in the API response from Fleet, otherwise it will be the Database ID of the VulnerabilityInstall record
|
|
vulnInstallRecordIdsThatNoLongerExistInFleetInstance.push(missingVulnInstallsByIds[installToCheck]);
|
|
}
|
|
}//∞
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${vulnInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed vulnerable software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
|
|
} else {
|
|
sails.log(`${vulnInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed vulnerable software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
|
|
let batchesOfVulnInstallsToMarkAsUninstalled = _.chunk(vulnInstallRecordIdsThatNoLongerExistInFleetInstance, 5000);
|
|
for(let batch of batchesOfVulnInstallsToMarkAsUninstalled){
|
|
await VulnerabilityInstall.update({ id: { in: batch }, uninstalledAt: 0 }).set({
|
|
uninstalledAt: Date.now()
|
|
});
|
|
}//∞
|
|
}
|
|
let criticalInstallRecordIdsThatNoLongerExistInFleetInstance = [];// « Create an array to store the database IDs of installs that Fleet did not return in the API response.
|
|
// This check is here to handles cases where vulnerable software is updated on a host, but the new version is affected by the same vulnerability as the old version.
|
|
for(let installToCheck in missingCriticalInstallsByHostIDs) {
|
|
if(missingCriticalInstallsByHostIDs[installToCheck]) {// This will be set to false if this install was included in the API response from Fleet, otherwise it will be the Database ID of the VulnerabilityInstall record
|
|
criticalInstallRecordIdsThatNoLongerExistInFleetInstance.push(missingCriticalInstallsByHostIDs[installToCheck]);
|
|
}
|
|
}//∞
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${criticalInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed critical software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
|
|
} else {
|
|
sails.log(`${criticalInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed critical software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
|
|
let batchesOfCriticalInstallsThatNoLongerExist = _.chunk(criticalInstallRecordIdsThatNoLongerExistInFleetInstance, 5000);
|
|
for(let batch of batchesOfCriticalInstallsThatNoLongerExist){
|
|
await CriticalInstall.destroy({ id: { in: batch } });
|
|
}//∞
|
|
}
|
|
// * * *
|
|
// Check for hosts in the database that are missing from the Fleet API.
|
|
let missingHosts = [];
|
|
await Host.stream()
|
|
.meta({batchSize: 1000})
|
|
.eachRecord(($host)=>{
|
|
if (!byHostFleetApidsSeenInLatestFleetScan[$host.fleetApid] && !byHostFleetApidsSeenInLatestCriticalSoftwareScan[$host.fleetApid]) {
|
|
missingHosts.push($host);
|
|
}
|
|
});//∞
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${missingHosts.length} previously-enrolled hosts seemingly unenrolled. (Fleet did not return them in the API this time.)`);
|
|
} else {
|
|
sails.log(`${missingHosts.length} previously-enrolled hosts seemingly unenrolled. (Fleet did not return them in the API this time.)`);
|
|
await VulnerabilityInstall.destroy({ host: { in: _.pluck(missingHosts,'id') } });
|
|
await CriticalInstall.destroy({ host: { in: _.pluck(missingHosts,'id') } });
|
|
await Host.destroy({ fleetApid: { in: _.pluck(missingHosts,'fleetApid') } });
|
|
}
|
|
// * * *
|
|
// Check for OperatingSystems that have not been marked as complaint are no longer in use
|
|
|
|
// Set the default values for the column names in our native queries to be lowercase for MySQL.
|
|
let operatingSystemColumnName = 'operatingsystem';
|
|
let isCompliantColumnName = 'iscompliant';
|
|
|
|
// If we're using sails-postgresql, we'll wrap the case-sensitive column names in double quotes.
|
|
if(sails.config.datastores.default.adapter === 'sails-postgresql'){
|
|
operatingSystemColumnName = '"operatingSystem"';
|
|
isCompliantColumnName = '"isCompliant"';
|
|
}
|
|
|
|
let nativeQueryToFindOperatingSystemsWithNoHosts =
|
|
`SELECT id, ${isCompliantColumnName}
|
|
FROM operatingsystem
|
|
WHERE id NOT IN (
|
|
SELECT ${operatingSystemColumnName}
|
|
FROM host
|
|
) AND ${isCompliantColumnName} = false;`;
|
|
|
|
let osWithNoHostsDatastoreResponse = await sails.sendNativeQuery(nativeQueryToFindOperatingSystemsWithNoHosts);
|
|
let osWithNoHosts = osWithNoHostsDatastoreResponse.rows;
|
|
|
|
// Check for operating systems that have been marked as compliant, but are no longer reported in the /os_versions API response.
|
|
// Since they have been marked as compliant on the patch-progress page, we wont delete the database record, we'll just update the host count to be 0.
|
|
let nativeQueryToFindCompliantOperatingSystemsWithNoHosts =
|
|
`SELECT id, ${isCompliantColumnName}
|
|
FROM operatingsystem
|
|
WHERE id NOT IN (
|
|
SELECT ${operatingSystemColumnName}
|
|
FROM host
|
|
) AND ${isCompliantColumnName} = true;`;
|
|
|
|
let compliantOsWithNoHostsDatastoreResponse = await sails.sendNativeQuery(nativeQueryToFindCompliantOperatingSystemsWithNoHosts);
|
|
let compliantOsWithNoHosts = compliantOsWithNoHostsDatastoreResponse.rows;
|
|
|
|
if(dry){
|
|
sails.log.warn(`Dry run: ${osWithNoHosts.length} operating system(s) no longer in use found, and ${compliantOsWithNoHosts.length} operating system(s) that have been previously marked as compliant no longer reported by the Fleet instance user found.`);
|
|
} else {
|
|
sails.log(`${osWithNoHosts.length} operating system(s) no longer in use found, and ${compliantOsWithNoHosts.length} complaint operating systems no longer reported by the Fleet instance. Saving....`);
|
|
// Delete non-compliant operating system records that are no longer installed.
|
|
await OperatingSystem.destroy({id: {in: _.pluck(osWithNoHosts, 'id')}});
|
|
// Update the host count for operating systems that have been marked as compliant, but are no longer installed on any host.
|
|
await OperatingSystem.update({id: {in: _.pluck(compliantOsWithNoHosts, 'id')}}).set({
|
|
lastReportedHostCount: 0
|
|
});
|
|
}
|
|
|
|
// Check for Vulnerabilities that have no associated Host or VulnerabilityInstall records.
|
|
let nativeQueryToFindVulnsWithNoAssociatedRecords =
|
|
`SELECT v.id
|
|
FROM vulnerability v
|
|
LEFT JOIN vulnerabilityinstall vi ON v.id = vi.vulnerability
|
|
WHERE vi.vulnerability IS NULL;`;
|
|
let rawResultFromDatabase = await sails.sendNativeQuery(nativeQueryToFindVulnsWithNoAssociatedRecords);
|
|
let vulnerabilityRecordIdsWithNoAssociatedRecords = rawResultFromDatabase.rows;
|
|
|
|
if(dry){
|
|
sails.log.warn(`Dry run: ${vulnerabilityRecordIdsWithNoAssociatedRecords.length} vulnerabilities affecting previously-enrolled hosts were found. (Fleet did not return them in the API this time.)`);
|
|
} else {
|
|
sails.log(`${vulnerabilityRecordIdsWithNoAssociatedRecords.length} vulnerabilities affecting previously-enrolled hosts were found. (Fleet did not return them in the API this time, and no associated Host or VulnerabilityInstall records were found.)`);
|
|
await Vulnerability.destroy({id: {in: _.pluck(vulnerabilityRecordIdsWithNoAssociatedRecords, 'id')}});
|
|
}
|
|
if(loggedWarningsFromThisScriptRun.length > 0) {
|
|
sails.log.warn(`During this run of the update-reports script ${loggedWarningsFromThisScriptRun.length} warning(s) were logged.`);
|
|
for(let warning of loggedWarningsFromThisScriptRun){
|
|
sails.log.warn(warning);
|
|
}
|
|
}
|
|
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
sails.log('Successfully completed scan and processing of',numVulnerableWaresProcessed,'vulnerable software items using Fleet.');
|
|
}
|
|
|
|
};
|