fleet/ee/bulk-operations-dashboard/api/controllers/software/edit-software.js
Scott Gress 556b79e2d2
Prevent Axios from fully buffering files uploaded from MSP dashboard (#24927)
for #24829 

See https://github.com/axios/axios/issues/1045 -- by default Axios
buffers uploaded files into memory fully, to support redirects. For
large file uploads this means we get OOM errors, especially when sending
to multiple teams. There's a few other optimizations we can put in place
here but in the short term we can fix the buffering issue by setting
`maxRedirects: 0` on the requests.

I tested this by adding an `onUploadProgress` handler to the Axios
request that dumps memory usage, and uploading a 209mb software file to
3 teams. Before the update, the readout ticket up continuously (the
first number is the # of bytes uploaded):

```
1540129 {rss: 161652736, heapTotal: 65880064, heapUsed: 55625552, external: 28411157, arrayBuffers: 24338844}
edit-software.js:177
1554313 {rss: 149254144, heapTotal: 65880064, heapUsed: 52445200, external: 25193635, arrayBuffers: 21121327}
edit-software.js:177
2339833 {rss: 151703552, heapTotal: 66404352, heapUsed: 52269280, external: 12664377, arrayBuffers: 8592064}
...a minute later...
192708641 {rss: 619323392, heapTotal: 95240192, heapUsed: 55320960, external: 618952429, arrayBuffers: 614879965}
edit-software.js:177
201523233 {rss: 634613760, heapTotal: 95240192, heapUsed: 58514992, external: 636581613, arrayBuffers: 632509154}
edit-software.js:177
209326677 {rss: 637399040, heapTotal: 95240192, heapUsed: 56800016, external: 639441633, arrayBuffers: 635369173}
```

so we start at ~161mb, and by the time we're done, we're using 637mb of
RAM. Render's free tier has a 250mb limit on apps.

With `maxRedirects: 0`, we see:

```
2669337 {rss: 151846912, heapTotal: 66404352, heapUsed: 53297400, external: 26446868, arrayBuffers: 22374419}
edit-software.js:177
2279929 {rss: 152641536, heapTotal: 66404352, heapUsed: 53453664, external: 27233300, arrayBuffers: 23160851}
edit-software.js:177
2228585 {rss: 153038848, heapTotal: 66404352, heapUsed: 53537096, external: 27626516, arrayBuffers: 23554067}
...a minute later...
209326677 {rss: 146989056, heapTotal: 92094464, heapUsed: 53802856, external: 14617518, arrayBuffers: 10545071}
edit-software.js:177
209326677 {rss: 153051136, heapTotal: 92094464, heapUsed: 55376336, external: 22447478, arrayBuffers: 18375026}
edit-software.js:177
209326677 {rss: 152129536, heapTotal: 92094464, heapUsed: 51857632, external: 22447478, arrayBuffers: 16540013}
```

showing that we start and finish with around the same amount of RAM
used.
2024-12-19 15:51:47 -06:00

463 lines
23 KiB
JavaScript

module.exports = {
friendlyName: 'Edit software',
description: 'Edits deployed software on a Fleet instance or undeployed software in the app\'s database',
files: ['newSoftware'],
inputs: {
newSoftware: {
type: 'ref',
description: 'The optional streaming bytes of a new software version.'
},
newTeamIds: {
type: ['string'],
description: 'The new teams that this software will be deployed to.'
},
software: {
type: {},
description: 'The software that will be editted.'
},
preInstallQuery: {
type: 'string',
},
installScript: {
type: 'string',
},
postInstallScript: {
type: 'string',
},
uninstallScript: {
type: 'string',
},
},
exits: {
wrongInstallerExtension: {
description: 'The provided replacement software\'s has the wrong extension.',
statusCode: 400,
},
softwareUploadFailed: {
description: 'The software upload failed'
},
softwareAlreadyExistsOnThisTeam: {
description: 'A software installer with this name already exists on the Fleet Instance',
},
couldNotReadVersion: {
description:'Fleet could not read version information from the provided software installer.'
},
softwareDeletionFailed: {
description: 'The specified software could not be deleted from the Fleet instance.',
statusCode: 409,
},
},
fn: async function ({newSoftware, newTeamIds, software, preInstallQuery, installScript, postInstallScript, uninstallScript}) {
if(newSoftware.isNoop) {
newSoftware.noMoreFiles();
newSoftware = undefined;
}
var WritableStream = require('stream').Writable;
// let { Readable } = require('stream');
let axios = require('axios');
// Cast the strings in the newTeamIds array to numbers.
if(newTeamIds){
newTeamIds = newTeamIds.map(Number);
} else {
newTeamIds = [];
}
let currentSoftwareTeamIds = _.pluck(software.teams, 'fleetApid');
// If the teams have changed, or a new installer package was provided, we'll need to upload the package to an s3 bucket to deploy it to other teams.
if(_.xor(newTeamIds, currentSoftwareTeamIds).length !== 0 || newSoftware) {
let softwareFd;
let softwareName;
let softwareMime;
if(software.teams && !newSoftware) {
// console.log('Editing deployed software!');
// This software is deployed.
// get software from Fleet instance and upload to s3.
let teamIdToGetInstallerFrom = software.teams[0].fleetApid;
let downloadApiUrl = `${sails.config.custom.fleetBaseUrl}/api/v1/fleet/software/titles/${software.fleetApid}/package?alt=media&team_id=${teamIdToGetInstallerFrom}`;
let softwareStream = await sails.helpers.http.getStream.with({
url: downloadApiUrl,
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`,
}
})
.intercept('non200Response', (error)=>{
return new Error(`When attempting to transfer the installer for ${software.name} to a new team on the Fleet instance, the Fleet isntance returned a non-200 response when a request was sent to get a download stream of the installer on team_id ${teamIdToGetInstallerFrom}. Full Error: ${require('util').inspect(error, {depth: 1})}`);
});
let tempUploadedSoftware = await sails.uploadOne(softwareStream, {bucket: sails.config.uploads.bucketWithPostfix});
softwareFd = tempUploadedSoftware.fd;
softwareName = software.name;
softwareMime = tempUploadedSoftware.type;
} else if(newSoftware) {
// If a new copy of the installer was uploaded, we'll
// console.log('replacing software package!');
let uploadedSoftware = await sails.uploadOne(newSoftware, {bucket: sails.config.uploads.bucketWithPostfix});
softwareFd = uploadedSoftware.fd;
softwareName = uploadedSoftware.filename;
softwareMime = uploadedSoftware.type;
let newSoftwareExtension = '.'+softwareName.split('.').pop();
let existingSoftwareExtension = '.'+software.name.split('.').pop();
if(newSoftwareExtension !== existingSoftwareExtension) {
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
throw {wrongInstallerExtension: `Couldn't edit ${software.name}. The selected package should be a ${existingSoftwareExtension} file`};
}
} else {
// console.log('Editing undeployed software!');
softwareFd = software.uploadFd;
softwareName = software.name;
softwareMime = software.uploadMime;
}
// Now apply the edits.
if(newTeamIds.length !== 0) {
let currentSoftwareTeamIds = _.pluck(software.teams, 'fleetApid') || [];
let addedTeams = _.difference(newTeamIds, currentSoftwareTeamIds);
let removedTeams = _.difference(currentSoftwareTeamIds, newTeamIds);
let unchangedTeamIds = _.difference(currentSoftwareTeamIds, removedTeams);
// for(let team of addedTeams) {
await sails.helpers.flow.simultaneouslyForEach(addedTeams, async (team)=>{
// console.log(`transfering ${software.name} to fleet instance for team id ${team}`);
// Send an api request to send the file to the Fleet server for each added team.
await sails.cp(softwareFd, {bucket: sails.config.uploads.bucketWithPostfix},
{
adapter: ()=>{
return {
ls: undefined,
rm: undefined,
read: undefined,
receive: (unusedOpts)=>{
// This `_write` method is invoked each time a new file is received
// from the Readable stream (Upstream) which is pumping filestreams
// into this receiver. (filename === `__newFile.filename`).
var receiver__ = WritableStream({ objectMode: true });
// Create a new drain (writable stream) to send through the individual bytes of this file.
receiver__._write = (__newFile, encoding, doneWithThisFile)=>{
let FormData = require('form-data');
let form = new FormData();
form.append('team_id', team);
form.append('install_script', installScript);
form.append('post_install_script', postInstallScript);
form.append('pre_install_query', preInstallQuery);
form.append('uninstall_script', uninstallScript);
form.append('software', __newFile, {
filename: software.name,
contentType: 'application/octet-stream'
});
(async ()=>{
await axios.postForm(`${sails.config.custom.fleetBaseUrl}/api/v1/fleet/software/package`, form, {
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`,
...form.getHeaders()
},
maxRedirects: 0
});
})()
.then(()=>{
// console.log('ok supposedly a file is finished uploading');
doneWithThisFile();
})
.catch((err)=>{
doneWithThisFile(err);
});
};//ƒ
return receiver__;
}
};
},
}
)
.intercept({response: {status: 409}}, async (error)=>{// handles errors related to duplicate software items.
if(!software.id) {// If the software does not have an ID, it not stored in the app's database/s3 bucket, so we can safely delete the file in s3.
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
}
return {'softwareAlreadyExistsOnThisTeam': error};
})
.intercept({name: 'AxiosError', response: {status: 400}}, async (error)=>{// Handles errors related to malformed installer packages
if(!software.id) {// If the software does not have an ID, it not stored in the app's database/s3 bucket, so we can safely delete the file in s3.
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
}
let axiosError = error;
if(axiosError.response.data) {
if(axiosError.response.data.errors && _.isArray(axiosError.response.data.errors)){
if(axiosError.response.data.errors[0] && axiosError.response.data.errors[0].reason) {
let errorMessageFromFleetInstance = axiosError.response.data.errors[0].reason;
if(_.startsWith(errorMessageFromFleetInstance, `Couldn't add. Fleet couldn't read the version`)){
return 'couldNotReadVersion';
} else {
sails.log.warn(`When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API. Error returned from Fleet API: ${errorMessageFromFleetInstance} \n Axios error: ${require('util').inspect(error, {depth: 3})}`);
return {'softwareUploadFailed': error};
}
}
}
}
sails.log.warn(`When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API, ${require('util').inspect(error, {depth: 3})}`);
return {'softwareUploadFailed': error};
})
.intercept(async (error)=>{
// Note: with this current behavior, all errors from this upload are currently swallowed and a softwareUploadFailed response is returned.
// FUTURE: Test to make sure that uploading duplicate software to a team results in a 409 response.
// Before handline errors, decide what to do about the file uploaded to s3, if this is undeployed software, we'll leave it alone, but if this was a temporary file created to transfer it between teams on the Fleet instance, we'll delete the file.
if(!software.id) {// If the software does not have an ID, it not stored in the app's database/s3 bucket, so we can safely delete the file in s3.
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
}
// Log a warning containing an error
sails.log.warn(`When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API, Full error: ${require('util').inspect(error, {depth: 3})}`);
return {'softwareUploadFailed': error};
});
// console.timeEnd(`transfering ${software.name} to fleet instance for team id ${team}`);
});
// }// After every new team this is deployed to.
if(newSoftware) {
// If a new installer package was provided, send patch requests to update the installer package on teams that it is already deployed to.
await sails.helpers.flow.simultaneouslyForEach(unchangedTeamIds, async (teamApid)=>{
// console.log(`Adding new version of ${softwareName} to teamId ${teamApid}`);
// console.time(`transfering ${software.name} to fleet instance for team id ${teamApid}`);
await sails.cp(softwareFd, {bucket: sails.config.uploads.bucketWithPostfix},
{
adapter: ()=>{
return {
ls: undefined,
rm: undefined,
read: undefined,
receive: (unusedOpts)=>{
// This `_write` method is invoked each time a new file is received
// from the Readable stream (Upstream) which is pumping filestreams
// into this receiver. (filename === `__newFile.filename`).
var receiver__ = WritableStream({ objectMode: true });
// Create a new drain (writable stream) to send through the individual bytes of this file.
receiver__._write = (__newFile, encoding, doneWithThisFile)=>{
let FormData = require('form-data');
let form = new FormData();
form.append('team_id', teamApid);
form.append('install_script', installScript);
form.append('post_install_script', postInstallScript);
form.append('pre_install_query', preInstallQuery);
form.append('uninstall_script', uninstallScript);
form.append('software', __newFile, {
filename: software.name,
contentType: 'application/octet-stream'
});
(async ()=>{
await axios.patch(`${sails.config.custom.fleetBaseUrl}/api/v1/fleet/software/titles/${software.fleetApid}/package`, form, {
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`,
...form.getHeaders()
},
maxRedirects: 0,
});
})()
.then(()=>{
// console.log('ok supposedly a file is finished uploading');
doneWithThisFile();
})
.catch((err)=>{
doneWithThisFile(err);
});
};//ƒ
return receiver__;
}
};
},
})
.intercept({response: {status: 409}}, async (error)=>{// handles errors related to duplicate software items.
if(!software.id) {// If the software does not have an ID, it not stored in the app's database/s3 bucket, so we can safely delete the file in s3.
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
}
return {'softwareAlreadyExistsOnThisTeam': error};
})
.intercept({name: 'AxiosError', response: {status: 400}}, async (error)=>{// Handles errors related to malformed installer packages
if(!software.id) {// If the software does not have an ID, it not stored in the app's database/s3 bucket, so we can safely delete the file in s3.
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
}
let axiosError = error;
if(axiosError.response.data) {
if(axiosError.response.data.errors && _.isArray(axiosError.response.data.errors)){
if(axiosError.response.data.errors[0] && axiosError.response.data.errors[0].reason) {
let errorMessageFromFleetInstance = axiosError.response.data.errors[0].reason;
if(_.startsWith(errorMessageFromFleetInstance, `Couldn't add. Fleet couldn't read the version`)){
return 'couldNotReadVersion';
} else {
sails.log.warn(`When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API. Error returned from Fleet API: ${errorMessageFromFleetInstance} \n Axios error: ${require('util').inspect(error, {depth: 3})}`);
return {'softwareUploadFailed': error};
}
}
}
}
sails.log.warn(`When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API, ${require('util').inspect(error, {depth: 3})}`);
return {'softwareUploadFailed': error};
})
.intercept(async (error)=>{
// Note: with this current behavior, all errors from this upload are currently swallowed and a softwareUploadFailed response is returned.
// FUTURE: Test to make sure that uploading duplicate software to a team results in a 409 response.
// Before handling errors, decide what to do about the file uploaded to s3, if this is undeployed software, we'll leave it alone, but if this was a temporary file created to transfer it between teams on the Fleet instance, we'll delete the file.
if(!software.id) {
await sails.rm(sails.config.uploads.prefixForFileDeletion+softwareFd);
}
// Log a warning containing an error
sails.log.warn(`When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API, ${require('util').inspect(error, {depth: 3})}`);
return {'softwareUploadFailed': error};
});
// console.timeEnd(`transfering ${software.name} to fleet instance for team id ${teamApid}`);
});// After every team the software is currently deployed to.
} else if(preInstallQuery !== software.preInstallQuery ||
installScript !== software.installScript ||
postInstallScript !== software.postInstallScript ||
uninstallScript !== software.uninstallScript) {
await sails.helpers.flow.simultaneouslyForEach(unchangedTeamIds, async (teamApid)=>{
await sails.helpers.http.sendHttpRequest.with({
method: 'PATCH',
baseUrl: sails.config.custom.fleetBaseUrl,
url: `/api/v1/fleet/software/titles/${software.fleetApid}/package?team_id=${teamApid}`,
enctype: 'multipart/form-data',
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`
},
body: {
team_id: teamApid, // eslint-disable-line camelcase
pre_install_query: preInstallQuery, // eslint-disable-line camelcase
install_script: installScript, // eslint-disable-line camelcase
post_install_script: postInstallScript, // eslint-disable-line camelcase
uninstall_script: uninstallScript, // eslint-disable-line camelcase
}
});
});
}
// Now delete the software from teams it was removed from.
for(let team of removedTeams) {
await sails.helpers.http.sendHttpRequest.with({
method: 'DELETE',
baseUrl: sails.config.custom.fleetBaseUrl,
url: `/api/v1/fleet/software/titles/${software.fleetApid}/available_for_install?team_id=${team}`,
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`,
}
})
.intercept({raw:{statusCode: 409}}, (error)=>{
// If the Fleet instance's returns a 409 response, then the software is configured to be installed as
// part of the macOS setup experience, and must be removed before it can be deleted via API requests.
return {softwareDeletionFailed: error};
});
}
// If the software had been previously undeployed, delete the installer in s3 and the db record.
if(software.id) {
await sails.rm(sails.config.uploads.prefixForFileDeletion+software.uploadFd);
await UndeployedSoftware.destroyOne({id: software.id});
}
} else if(software.teams && newTeamIds.length === 0) {
// If this is a deployed software that is being unassigned, save information about the uploaded file in our s3 bucket.
for(let team of software.teams) {
// Now delete the software on the Fleet instance.
await sails.helpers.http.sendHttpRequest.with({
method: 'DELETE',
baseUrl: sails.config.custom.fleetBaseUrl,
url: `/api/v1/fleet/software/titles/${software.fleetApid}/available_for_install?team_id=${team.fleetApid}`,
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`,
}
})
.intercept({raw:{statusCode: 409}}, (error)=>{
// If the Fleet instance's returns a 409 response, then the software is configured to be installed as
// part of the macOS setup experience, and must be removed before it can be deleted via API requests.
return {softwareDeletionFailed: error};
});
}
if(newSoftware) {
await UndeployedSoftware.create({
uploadFd: softwareFd,
uploadMime: softwareMime,
name: softwareName,
platform: software.platform,
postInstallScript,
preInstallQuery,
installScript,
uninstallScript,
});
} else {
// Save the information about the undeployed software in the app's DB.
await UndeployedSoftware.create({
uploadFd: softwareFd,
uploadMime: softwareMime,
name: software.name,
platform: software.platform,
postInstallScript,
preInstallQuery,
installScript,
uninstallScript,
});
}
} else {
// console.log('updating existing db record!');
await UndeployedSoftware.updateOne({id: software.id}).set({
name: softwareName,
uploadMime: softwareMime,
uploadFd: softwareFd,
preInstallQuery,
installScript,
postInstallScript,
uninstallScript,
});
// console.log('removing old stored copy of '+softwareName);
await sails.rm(sails.config.uploads.prefixForFileDeletion+software.uploadFd);
}
} else if(preInstallQuery !== software.preInstallQuery ||
installScript !== software.installScript ||
postInstallScript !== software.postInstallScript ||
uninstallScript !== software.uninstallScript) {
// PATCH /api/v1/fleet/software/titles/:title_id/package
if(newTeamIds.length !== 0) {
for(let teamApid of newTeamIds){
await sails.helpers.http.sendHttpRequest.with({
method: 'PATCH',
baseUrl: sails.config.custom.fleetBaseUrl,
url: `/api/v1/fleet/software/titles/${software.fleetApid}/package?team_id=${teamApid}`,
enctype: 'multipart/form-data',
headers: {
Authorization: `Bearer ${sails.config.custom.fleetApiToken}`
},
body: {
team_id: teamApid, // eslint-disable-line camelcase
pre_install_query: preInstallQuery, // eslint-disable-line camelcase
install_script: installScript, // eslint-disable-line camelcase
post_install_script: postInstallScript, // eslint-disable-line camelcase
uninstall_script: uninstallScript, // eslint-disable-line camelcase
}
});
}
} else if(software.id) {
await UndeployedSoftware.updateOne({id: software.id}).set({
preInstallQuery,
installScript,
postInstallScript,
uninstallScript,
});
}
}
return;
}
};