mirror of
https://github.com/fleetdm/fleet
synced 2026-04-21 21:47:20 +00:00
359 lines
21 KiB
JavaScript
359 lines
21 KiB
JavaScript
module.exports = {
|
|
|
|
|
|
friendlyName: 'Update critical software',
|
|
|
|
|
|
description: '',
|
|
|
|
inputs: {
|
|
dry: {
|
|
description: 'Do a dry run instead of actually modifying the database?',
|
|
type: 'boolean',
|
|
defaultsTo: false
|
|
},
|
|
},
|
|
|
|
fn: async function ({dry}) {
|
|
let assert = require('assert');
|
|
// console.time('Update reports script');
|
|
|
|
// ┌─┐┌─┐┌┬┐┬ ┬┌─┐┬─┐ ┌─┐┌┐┌┌┬┐ ┌─┐┬─┐┌─┐┌─┐┌─┐┌─┐┌─┐ ┌─┐┬ ┌─┐┌─┐┌┬┐ ┌─┐┌─┐┬ ┌┬┐┌─┐┌┬┐┌─┐
|
|
// │ ┬├─┤ │ ├─┤├┤ ├┬┘ ├─┤│││ ││ ├─┘├┬┘│ ││ ├┤ └─┐└─┐ ├┤ │ ├┤ ├┤ │ ├─┤├─┘│ ││├─┤ │ ├─┤
|
|
// └─┘┴ ┴ ┴ ┴ ┴└─┘┴└─ ┴ ┴┘└┘─┴┘ ┴ ┴└─└─┘└─┘└─┘└─┘└─┘ └ ┴─┘└─┘└─┘ ┴ ┴ ┴┴ ┴ ─┴┘┴ ┴ ┴ ┴ ┴
|
|
if (!sails.config.custom.fleetBaseUrl || !sails.config.custom.fleetApiToken) {
|
|
throw new Error('sails.config.custom.fleetBaseUrl and sails.config.custom.fleetApiToken must both be provided.');
|
|
}
|
|
let fleetBaseUrl = sails.config.custom.fleetBaseUrl;
|
|
let headers = {
|
|
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`
|
|
};
|
|
if (sails.config.custom.fleetApiOptionalCookie) {
|
|
headers['Cookie'] = sails.config.custom.fleetApiOptionalCookie;
|
|
}
|
|
sails.log('Running custom shell script... (`sails run update-critical-software`)');
|
|
|
|
|
|
let byHostFleetApidsSeenInLatestCriticalSoftwareScan = {};
|
|
|
|
|
|
|
|
let allKnownExistingCriticalInstalls = await CriticalInstall.find({select: ['host', 'fleetApid',]});
|
|
|
|
|
|
// Build a dictionary where every existing VulnerabilityInstall record has a unique string as it's key that we'll use to check if a install records we're processing exists already.
|
|
|
|
let existingCriticalInstallsByHostIDs = {};
|
|
// Build a second dictionary to track existing VulnerabilityInstall records that are not included in the Fleet API response this run.
|
|
|
|
let missingCriticalInstallsByHostIDs = {};
|
|
|
|
for(let $install of allKnownExistingCriticalInstalls) {
|
|
existingCriticalInstallsByHostIDs[`${$install.fleetApid}|${$install.host}`] = true;// « ex: {'140|2146':true, '135|6729':true, ...}
|
|
missingCriticalInstallsByHostIDs[`${$install.fleetApid}|${$install.host}`] = $install.id;// « ex: {'140|2146':true, '135|6729':true, ...}
|
|
}//∞
|
|
|
|
// ██████╗ ███████╗ ██╗ ██╗███████╗██████╗ ███████╗██╗ ██████╗ ███╗ ██╗███████╗
|
|
// ██╔═══██╗██╔════╝ ██║ ██║██╔════╝██╔══██╗██╔════╝██║██╔═══██╗████╗ ██║██╔════╝
|
|
// ██║ ██║███████╗ ██║ ██║█████╗ ██████╔╝███████╗██║██║ ██║██╔██╗ ██║███████╗
|
|
// ██║ ██║╚════██║ ╚██╗ ██╔╝██╔══╝ ██╔══██╗╚════██║██║██║ ██║██║╚██╗██║╚════██║
|
|
// ╚██████╔╝███████║ ╚████╔╝ ███████╗██║ ██║███████║██║╚██████╔╝██║ ╚████║███████║
|
|
// ╚═════╝ ╚══════╝ ╚═══╝ ╚══════╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚══════╝
|
|
//
|
|
let osVersionNamesByHostCount = {};
|
|
let allKnownOperatingSystems = await OperatingSystem.find();
|
|
for(let osRecord of allKnownOperatingSystems){
|
|
osVersionNamesByHostCount[`${osRecord.fullName}`] = osRecord.lastReportedHostCount;
|
|
}
|
|
|
|
let osVersionsResponseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/os_versions',
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}]);
|
|
assert(undefined === osVersionsResponseData.os_versions || _.isArray(osVersionsResponseData.os_versions));
|
|
|
|
let currentOSVersions = osVersionsResponseData.os_versions;
|
|
let osVersionsToReport = [];
|
|
let osVersionsToUpdate = [];
|
|
for(let os of currentOSVersions) {
|
|
let osToReport = {
|
|
name: os.name_only,
|
|
fullName: os.name,
|
|
platform: os.platform !== '' ? os.platform : os.name,
|
|
versionName: os.version,
|
|
lastReportedHostCount: os.hosts_count
|
|
};
|
|
// console.log(os.name);
|
|
// console.log(osVersionNamesByHostCount, os.hosts_count);
|
|
if(osVersionNamesByHostCount[os.name] === undefined) {
|
|
osVersionsToReport.push(osToReport);
|
|
} else if(osVersionNamesByHostCount[os.name] !== os.hosts_count) {
|
|
}
|
|
osVersionsToUpdate.push(osToReport);
|
|
}
|
|
|
|
let nativeQueryToFindOperatingSystemsWithNoHosts =
|
|
`SELECT id, iscompliant
|
|
FROM operatingsystem
|
|
WHERE id NOT IN (
|
|
SELECT operatingsystem
|
|
FROM host
|
|
) AND iscompliant = false;`;
|
|
let osWithNoHostsDatastoreResponse = await sails.sendNativeQuery(nativeQueryToFindOperatingSystemsWithNoHosts);
|
|
let osWithNoHosts = osWithNoHostsDatastoreResponse.rows;
|
|
|
|
|
|
|
|
if(dry){
|
|
// console.log(osVersionsToReport);
|
|
// console.log(osVersionsToUpdate);
|
|
|
|
sails.log.warn(`Dry run: ${osVersionsToReport.length} new operating systems, and changes to ${osVersionsToUpdate.length} existing operating system(s) detected.`);
|
|
} else {
|
|
sails.log(`Creating ${osVersionsToReport.length} new OperatingSystem records, saving updates to ${osVersionsToUpdate.length}, and removing ${osWithNoHosts.length} operating systems.`);
|
|
await OperatingSystem.createEach(osVersionsToReport);
|
|
for(let osRecordToUpdate of osVersionsToUpdate){
|
|
await OperatingSystem.updateOne({fullName: osRecordToUpdate.fullName}).set({lastReportedHostCount: osRecordToUpdate.lastReportedHostCount});
|
|
}
|
|
await OperatingSystem.destroy({id: {in: _.pluck(osWithNoHosts, 'id')}});
|
|
}
|
|
|
|
let allOsRecords = await OperatingSystem.find();
|
|
|
|
|
|
|
|
// ██████╗██████╗ ██╗████████╗██╗ ██████╗ █████╗ ██╗
|
|
// ██╔════╝██╔══██╗██║╚══██╔══╝██║██╔════╝██╔══██╗██║
|
|
// ██║ ██████╔╝██║ ██║ ██║██║ ███████║██║
|
|
// ██║ ██╔══██╗██║ ██║ ██║██║ ██╔══██║██║
|
|
// ╚██████╗██║ ██║██║ ██║ ██║╚██████╗██║ ██║███████╗
|
|
// ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚══════╝
|
|
//
|
|
// ███████╗ ██████╗ ███████╗████████╗██╗ ██╗ █████╗ ██████╗ ███████╗
|
|
// ██╔════╝██╔═══██╗██╔════╝╚══██╔══╝██║ ██║██╔══██╗██╔══██╗██╔════╝
|
|
// ███████╗██║ ██║█████╗ ██║ ██║ █╗ ██║███████║██████╔╝█████╗
|
|
// ╚════██║██║ ██║██╔══╝ ██║ ██║███╗██║██╔══██║██╔══██╗██╔══╝
|
|
// ███████║╚██████╔╝██║ ██║ ╚███╔███╔╝██║ ██║██║ ██║███████╗
|
|
// ╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚══╝╚══╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝
|
|
//
|
|
let QUERIES_TO_GET_CRITICAL_SOFTWARE = [
|
|
{query: 'Safari.app', type: 'safari'},// Safari on macOS
|
|
{query: 'Firefox.app', type: 'firefox'},// Safari on macOS
|
|
{query: 'Google Chrome', type: 'chrome'},// Safari on macOS
|
|
{query: 'Google Chrome.app', type: 'chrome'},// Safari on macOS
|
|
{query: 'Firefox', type: 'firefox'},// Firefox on Linux
|
|
{query: 'Mozilla Firefox', type: 'firefox'},// Firefox on Windows?
|
|
{query: 'Flash player.app', type: 'flash'}, // Flash on macOS
|
|
{query: 'Flash', type: 'flash'},// Flash on ?????
|
|
{query: 'Microsoft Excel', type: 'microsoftOffice'},
|
|
{query: 'Microsoft Powerpoint', type: 'microsoftOffice'},
|
|
{query: 'Microsoft Word', type: 'microsoftOffice'},
|
|
{query: 'Microsoft Outlook', type: 'microsoftOffice'},
|
|
];
|
|
let potentialCriticalInstalls = [];
|
|
await sails.helpers.flow.forEach(QUERIES_TO_GET_CRITICAL_SOFTWARE, async (softwareToLookFor)=>{
|
|
// Send a request to the software endpoint for each query to get a list of software isntalled for each query.
|
|
|
|
let criticalWares;
|
|
{
|
|
let responseData = await sails.helpers.http.get.with({
|
|
url: '/api/v1/fleet/software',
|
|
data: { query: softwareToLookFor.query },
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}]); //
|
|
assert(undefined === responseData.software || _.isArray(responseData.software));
|
|
if (!responseData.software) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return;
|
|
}//•
|
|
if(softwareToLookFor.type === 'flash') {
|
|
criticalWares = responseData.software.filter((software)=>{
|
|
if(software.source !== 'firefox_addons' && software.source !== 'chrome_extensions'){
|
|
return software.name.toLowerCase() === softwareToLookFor.query.toLowerCase();
|
|
}
|
|
});
|
|
} else if(softwareToLookFor.type === 'chrome') {
|
|
criticalWares = responseData.software.filter((software)=>{
|
|
if(software.source !== 'firefox_addons' && software.source !== 'chrome_extensions' && !software.name.match(/helper/gi)) {
|
|
return _.startsWith(software.name.toLowerCase(), softwareToLookFor.query.toLowerCase());
|
|
}
|
|
});
|
|
} else {
|
|
criticalWares = responseData.software.filter((software)=>{
|
|
if(software.source !== 'firefox_addons' && software.source !== 'chrome_extensions'){
|
|
return _.startsWith(software.name.toLowerCase(), softwareToLookFor.query.toLowerCase());
|
|
}
|
|
});
|
|
}
|
|
}//∫
|
|
|
|
let hostApidsBySoftwareVersionApid = {};
|
|
await sails.helpers.flow.simultaneouslyForEach(criticalWares, async(ware)=>{
|
|
// Get hosts with this version of software installed.
|
|
// [?] https://fleetdm.com/docs/using-fleet/rest-api#list-hosts
|
|
// # requests == O(SOFTWARE_VERSIONS_PAGE_SIZE * (vulnerableSoftware / SOFTWARE_VERSIONS_PAGE_SIZE))
|
|
|
|
let hosts = [];
|
|
{
|
|
let hostsPage = 0;
|
|
let HOSTS_PAGE_SIZE = 100;
|
|
await sails.helpers.flow.until( async()=>{
|
|
// sails.log(`sending request for page ${hostsPage} for ${ware.name}.`)
|
|
let responseData = await sails.helpers.http.get.with({// FUTURE: Paginate this using `.until()` for further scalability, as needed.
|
|
url: '/api/v1/fleet/hosts',
|
|
data: { software_id: ware.id, page: hostsPage, per_page: HOSTS_PAGE_SIZE, order_key: 'hostname', order_direction: 'asc' }, //eslint-disable-line camelcase
|
|
baseUrl: fleetBaseUrl,
|
|
headers
|
|
})
|
|
.timeout(120000)
|
|
.retry(['requestFailed', {name: 'TimeoutError'}])
|
|
.intercept({raw:{statusCode: 404}} , (error)=>{
|
|
return new Error(`When sending a request to the '/api/v1/fleet/hosts' API endpoint to get a filtered array of hosts with ${ware.name} ${ware.version} installed (software ID: ${ware.id}), the Fleet instance returned a 404 response when we expected it to return an array of ${ware.hosts_count} host(s).\n Response from Fleet instance: ${error.raw.body}`);
|
|
});
|
|
// sails.log.warn(`got a response for page ${hostsPage} for ${ware.name}.`);
|
|
if (!responseData.hosts) {// When pages of results are exhausted, bail. (`responseData.software` is absent in that case)
|
|
return true;
|
|
}//•
|
|
assert(_.isArray(responseData.hosts));
|
|
assert(responseData.hosts.every((host) => _.isNumber(host.id)));
|
|
hosts = hosts.concat(responseData.hosts);
|
|
// console.timeEnd(`Page ${page} of vulnerable software from Fleet API`);
|
|
hostsPage++;
|
|
if(responseData.hosts.length !== HOSTS_PAGE_SIZE){
|
|
return true;
|
|
}
|
|
});
|
|
}//∫
|
|
|
|
|
|
|
|
hostApidsBySoftwareVersionApid[ware.id] = _.pluck(hosts, 'id');// Keep track of which hosts have this software installed, for use below w/ vulns.
|
|
for (let host of hosts) {
|
|
let hostsOperatingSystem = _.find(allOsRecords, {'fullName': host.os_version});
|
|
// console.log(hostsOperatingSystem);
|
|
if(!hostsOperatingSystem){
|
|
hostsOperatingSystem = _.find(allOsRecords, {'fullName': 'Microsoft '+host.os_version});
|
|
if(!hostsOperatingSystem){
|
|
throw new Error(`Host's operating system not found in Operating System records`);
|
|
}
|
|
}
|
|
byHostFleetApidsSeenInLatestCriticalSoftwareScan[host.id] = {
|
|
displayName: _.trim(host.display_name.replace(/,/g, '')),// Trim leading and trailing whitespace, and remove commas from Host names.
|
|
teamDisplayName: host.team_name !== null ? host.team_name : 'No team',
|
|
teamApid: host.team_id !== null ? host.team_id : 0, // Note: If the host is not a member of any team, we'll set this to 0
|
|
fleetApid: host.id,
|
|
hardwareSerialNumber: host.hardware_serial !== '' ? host.hardware_serial : 'N/A',// Note: If a host does not have a hardware_serial value (e.g., ChromeOS hosts), this value will be set to 'N/A'.
|
|
uuid: host.uuid,
|
|
operatingSystem: hostsOperatingSystem.id,
|
|
};//∞
|
|
}
|
|
}).intercept((err)=>{
|
|
sails.log.error('An error occurred in side the iteratee of a flow control function. Due to an unresolved issue, these errors can be hard to dissect without code changes. Just in case, this log message seeks to dump as much info as possible!\n','err',err,'\nerr.raw', err.raw);
|
|
// // Since .raw isn't displayed in output again in Node ≥12, try to make the error better.
|
|
if (err.code === 'E_INTERNAL_ERROR') {
|
|
return err.raw;
|
|
} else {
|
|
return err;
|
|
}
|
|
});//∞ </each software version>
|
|
|
|
let hostRecordsToUpdate = [];
|
|
// Unrecognized hosts? Save 'em to the database.
|
|
let newRecordsForUnrecognizedHosts = []; {
|
|
let recognizedHosts = await Host.find({ fleetApid: { in: Object.keys(byHostFleetApidsSeenInLatestCriticalSoftwareScan).map((key)=>Number(key)) } });
|
|
let unrecognizedHostApids = _.difference(Object.keys(byHostFleetApidsSeenInLatestCriticalSoftwareScan).map((key)=>Number(key)), _.pluck(recognizedHosts, 'fleetApid'));
|
|
assert(unrecognizedHostApids.every((apid) => _.isNumber(apid)));
|
|
for (let apid of unrecognizedHostApids) {
|
|
newRecordsForUnrecognizedHosts.push(byHostFleetApidsSeenInLatestCriticalSoftwareScan[apid]);
|
|
}//∞
|
|
for(let host of recognizedHosts) {
|
|
let potentialUpdatesToHostRecord = byHostFleetApidsSeenInLatestCriticalSoftwareScan[host.fleetApid];
|
|
// If the host's displayName, teamApid, teamDisplayName, or operating system has changed, we'll update the host record with the latest information.
|
|
if(potentialUpdatesToHostRecord.teamApid !== host.teamApid
|
|
|| potentialUpdatesToHostRecord.teamDisplayName !== host.teamDisplayName
|
|
|| potentialUpdatesToHostRecord.operatingSystem !== host.operatingSystem
|
|
|| potentialUpdatesToHostRecord.displayName !== host.displayName)
|
|
{
|
|
// Add the database id of this host to the record to update, and add it to the array of hostRecordsToUpdate.
|
|
potentialUpdatesToHostRecord.id = host.id;
|
|
hostRecordsToUpdate.push(potentialUpdatesToHostRecord);
|
|
|
|
}
|
|
|
|
}
|
|
}//∫
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${newRecordsForUnrecognizedHosts.length} hosts were seemingly enrolled. (Fleet returned them in the API.)`);
|
|
sails.log.warn(`Dry run: ${hostRecordsToUpdate.length} hosts will be updated with new information. (Fleet returned them in the API.)`);
|
|
// console.log(`would have created ${newRecordsForUnrecognizedHosts.length}:`,require('util').inspect(newRecordsForUnrecognizedHosts,{depth:null}));
|
|
} else {
|
|
sails.log(`Creating ${newRecordsForUnrecognizedHosts.length} host records… `);
|
|
await Host.createEach(newRecordsForUnrecognizedHosts);
|
|
sails.log(`Updating ${hostRecordsToUpdate.length} host records… `);
|
|
for(let hostUpdate of hostRecordsToUpdate){
|
|
await Host.updateOne({id: hostUpdate.id}).set(_.omit(hostUpdate, 'id'));
|
|
}
|
|
}
|
|
|
|
|
|
for (let ware of criticalWares) {
|
|
// console.time(`processed results for ware: ${ware.id}`);
|
|
let $hosts = await Host.find({ fleetApid: { in: hostApidsBySoftwareVersionApid[ware.id] }}).populate('operatingSystem');// FUTURE: optimize (O(n) queries)
|
|
// console.log(ware.id+': '+$vulns.length +' vulnerabilities, '+$hosts.length +' hosts');
|
|
for (let $host of $hosts) {
|
|
// If a VulnerabilityInstall record exists that has the same host and vulnerability as the one we detected, we'll ignore it.
|
|
let criticalInstallExists = existingCriticalInstallsByHostIDs[`${ware.id}|${$host.id}`];
|
|
missingCriticalInstallsByHostIDs[`${ware.id}|${$host.id}`] = false;// « ex: {'140|2146':true, '135|6729':true, ...}
|
|
if(!criticalInstallExists) {
|
|
potentialCriticalInstalls.push({
|
|
installedAt: Date.now(),
|
|
host: $host.id,
|
|
fleetApid: Number(ware.id),
|
|
softwareName: ware.name.replace(/.app$/i, ''),
|
|
versionName: ware.version,
|
|
softwareType: softwareToLookFor.type,
|
|
platform: $host.operatingSystem.platform,
|
|
});
|
|
// Add this install to the existingVulnInstallsByHostAndVulnIDs dictionary
|
|
existingCriticalInstallsByHostIDs[`${ware.id}|${$host.id}`] = true;
|
|
}
|
|
}//∞
|
|
// console.timeEnd(`processed results for ware: ${ware.id}`);
|
|
}//∞
|
|
});
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${potentialCriticalInstalls.length} potential critical installs are available.`);
|
|
} else {
|
|
sails.log(`Detected ${potentialCriticalInstalls.length} changes to critical software installations. Saving...`);
|
|
let batchedPotentialCriticalInstalls = _.chunk(potentialCriticalInstalls, 1000);
|
|
for(let batch of batchedPotentialCriticalInstalls){
|
|
await CriticalInstall.createEach(batch);
|
|
}
|
|
}
|
|
|
|
let criticalInstallRecordIdsThatNoLongerExistInFleetInstance = [];// « Create an array to store the database IDs of installs that Fleet did not return in the API response.
|
|
// This check is here to handles cases where vulnerable software is updated on a host, but the new version is affected by the same vulnerability as the old version.
|
|
for(let installToCheck in missingCriticalInstallsByHostIDs) {
|
|
if(missingCriticalInstallsByHostIDs[installToCheck]) {// This will be set to false if this install was included in the API response from Fleet, otherwise it will be the Database ID of the VulnerabilityInstall record
|
|
criticalInstallRecordIdsThatNoLongerExistInFleetInstance.push(missingCriticalInstallsByHostIDs[installToCheck]);
|
|
}
|
|
}//∞
|
|
|
|
if (dry) {
|
|
sails.log.warn(`Dry run: ${criticalInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed critical software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
|
|
} else {
|
|
sails.log(`${criticalInstallRecordIdsThatNoLongerExistInFleetInstance.length} previously-installed critical software items were seemingly uninstalled. (Fleet did not return them in the API this time.)`);
|
|
await CriticalInstall.destroy({ id: { in: criticalInstallRecordIdsThatNoLongerExistInFleetInstance } });
|
|
}
|
|
},
|
|
|
|
|
|
|
|
};
|