2024-10-15 15:17:05 +00:00
module . exports = {
friendlyName : 'Upload software' ,
description : '' ,
files : [ 'newSoftware' ] ,
inputs : {
newSoftware : {
type : 'ref' ,
description : 'An Upstream with an incoming file upload.' ,
required : true ,
} ,
teams : {
type : [ 'string' ] ,
description : 'An array of team IDs that this profile will be added to'
}
} ,
exits : {
success : {
outputDescription : 'The new software has been uploaded' ,
outputType : { } ,
} ,
softwareAlreadyExistsOnThisTeam : {
description : 'A software with this name already exists on the Fleet Instance' ,
statusCode : 409 ,
} ,
2024-10-22 02:13:06 +00:00
softwareUploadFailed : {
description : 'An unexpected error occurred communicating with the Fleet API'
2024-12-03 04:28:03 +00:00
} ,
couldNotReadVersion : {
description : 'Fleet could not read version information from the provided software installer.'
2024-10-22 02:13:06 +00:00
}
2024-10-15 15:17:05 +00:00
} ,
fn : async function ( { newSoftware , teams } ) {
let uploadedSoftware ;
if ( ! teams ) {
uploadedSoftware = await sails . uploadOne ( newSoftware , { bucket : sails . config . uploads . bucketWithPostfix } ) ;
let datelessFilename = uploadedSoftware . filename . replace ( /^\d{4}-\d{2}-\d{2}\s/ , '' ) ;
// Build a dictonary of information about this software to return to the softwares page.
let newSoftwareInfo = {
name : datelessFilename ,
platform : _ . endsWith ( datelessFilename , '.deb' ) ? 'Linux' : _ . endsWith ( datelessFilename , '.pkg' ) ? 'macOS' : 'Windows' ,
createdAt : Date . now ( ) ,
uploadFd : uploadedSoftware . fd ,
uploadMime : uploadedSoftware . type ,
} ;
await UndeployedSoftware . create ( newSoftwareInfo ) ;
} else {
2024-10-22 02:13:06 +00:00
uploadedSoftware = await sails . uploadOne ( newSoftware , { bucket : sails . config . uploads . bucketWithPostfix } ) ;
2024-10-15 15:17:05 +00:00
for ( let teamApid of teams ) {
var WritableStream = require ( 'stream' ) . Writable ;
await sails . cp ( uploadedSoftware . fd , { bucket : sails . config . uploads . bucketWithPostfix } , {
adapter : ( ) => {
return {
ls : undefined ,
rm : undefined ,
read : undefined ,
receive : ( unusedOpts ) => {
// This `_write` method is invoked each time a new file is received
// from the Readable stream (Upstream) which is pumping filestreams
// into this receiver. (filename === `__newFile.filename`).
var receiver _ _ = WritableStream ( { objectMode : true } ) ;
// Create a new drain (writable stream) to send through the individual bytes of this file.
receiver _ _ . _write = ( _ _newFile , encoding , doneWithThisFile ) => {
let axios = require ( 'axios' ) ;
let FormData = require ( 'form-data' ) ;
let form = new FormData ( ) ;
form . append ( 'team_id' , teamApid ) ;
form . append ( 'software' , _ _newFile , {
filename : uploadedSoftware . filename ,
contentType : 'application/octet-stream'
} ) ;
( async ( ) => {
2024-12-03 21:42:42 +00:00
await axios . postForm ( ` ${ sails . config . custom . fleetBaseUrl } /api/v1/fleet/software/package ` , form , {
2024-10-15 15:17:05 +00:00
headers : {
Authorization : ` Bearer ${ sails . config . custom . fleetApiToken } ` ,
... form . getHeaders ( )
} ,
Prevent Axios from fully buffering files uploaded from MSP dashboard (#24927)
for #24829
See https://github.com/axios/axios/issues/1045 -- by default Axios
buffers uploaded files into memory fully, to support redirects. For
large file uploads this means we get OOM errors, especially when sending
to multiple teams. There's a few other optimizations we can put in place
here but in the short term we can fix the buffering issue by setting
`maxRedirects: 0` on the requests.
I tested this by adding an `onUploadProgress` handler to the Axios
request that dumps memory usage, and uploading a 209mb software file to
3 teams. Before the update, the readout ticket up continuously (the
first number is the # of bytes uploaded):
```
1540129 {rss: 161652736, heapTotal: 65880064, heapUsed: 55625552, external: 28411157, arrayBuffers: 24338844}
edit-software.js:177
1554313 {rss: 149254144, heapTotal: 65880064, heapUsed: 52445200, external: 25193635, arrayBuffers: 21121327}
edit-software.js:177
2339833 {rss: 151703552, heapTotal: 66404352, heapUsed: 52269280, external: 12664377, arrayBuffers: 8592064}
...a minute later...
192708641 {rss: 619323392, heapTotal: 95240192, heapUsed: 55320960, external: 618952429, arrayBuffers: 614879965}
edit-software.js:177
201523233 {rss: 634613760, heapTotal: 95240192, heapUsed: 58514992, external: 636581613, arrayBuffers: 632509154}
edit-software.js:177
209326677 {rss: 637399040, heapTotal: 95240192, heapUsed: 56800016, external: 639441633, arrayBuffers: 635369173}
```
so we start at ~161mb, and by the time we're done, we're using 637mb of
RAM. Render's free tier has a 250mb limit on apps.
With `maxRedirects: 0`, we see:
```
2669337 {rss: 151846912, heapTotal: 66404352, heapUsed: 53297400, external: 26446868, arrayBuffers: 22374419}
edit-software.js:177
2279929 {rss: 152641536, heapTotal: 66404352, heapUsed: 53453664, external: 27233300, arrayBuffers: 23160851}
edit-software.js:177
2228585 {rss: 153038848, heapTotal: 66404352, heapUsed: 53537096, external: 27626516, arrayBuffers: 23554067}
...a minute later...
209326677 {rss: 146989056, heapTotal: 92094464, heapUsed: 53802856, external: 14617518, arrayBuffers: 10545071}
edit-software.js:177
209326677 {rss: 153051136, heapTotal: 92094464, heapUsed: 55376336, external: 22447478, arrayBuffers: 18375026}
edit-software.js:177
209326677 {rss: 152129536, heapTotal: 92094464, heapUsed: 51857632, external: 22447478, arrayBuffers: 16540013}
```
showing that we start and finish with around the same amount of RAM
used.
2024-12-19 21:51:47 +00:00
maxRedirects : 0
2024-10-15 15:17:05 +00:00
} ) ;
} ) ( )
. then ( ( ) => {
// console.log('ok supposedly a file is finished uploading');
doneWithThisFile ( ) ;
} )
. catch ( ( err ) => {
doneWithThisFile ( err ) ;
} ) ;
} ; //ƒ
return receiver _ _ ;
}
} ;
}
} )
2024-12-03 04:28:03 +00:00
. intercept ( { response : { status : 409 } } , async ( error ) => { // handles errors related to duplicate software items.
2024-10-22 02:13:06 +00:00
await sails . rm ( sails . config . uploads . prefixForFileDeletion + uploadedSoftware . fd ) ;
2024-10-15 15:17:05 +00:00
return { 'softwareAlreadyExistsOnThisTeam' : error } ;
2024-10-22 02:13:06 +00:00
} )
2024-12-03 04:28:03 +00:00
. intercept ( { name : 'AxiosError' , response : { status : 400 } } , async ( error ) => { // Handles errors related to malformed installer packages
await sails . rm ( sails . config . uploads . prefixForFileDeletion + uploadedSoftware . fd ) ;
let axiosError = error ;
if ( axiosError . response . data ) {
if ( axiosError . response . data . errors && _ . isArray ( axiosError . response . data . errors ) ) {
if ( axiosError . response . data . errors [ 0 ] && axiosError . response . data . errors [ 0 ] . reason ) {
let errorMessageFromFleetInstance = axiosError . response . data . errors [ 0 ] . reason ;
if ( _ . startsWith ( errorMessageFromFleetInstance , ` Couldn't add. Fleet couldn't read the version ` ) ) {
return 'couldNotReadVersion' ;
} else {
2024-12-03 20:25:11 +00:00
sails . log . warn ( ` When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API. Error returned from Fleet API: ${ errorMessageFromFleetInstance } \n Axios error: ${ require ( 'util' ) . inspect ( error , { depth : 3 } )} ` ) ;
2024-12-03 04:28:03 +00:00
return { 'softwareUploadFailed' : error } ;
}
}
}
}
sails . log . warn ( ` When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API, ${ require ( 'util' ) . inspect ( error , { depth : 3 } )} ` ) ;
return { 'softwareUploadFailed' : error } ;
} )
. intercept ( { name : 'AxiosError' } , async ( error ) => { // Handles any other error.
2024-10-22 02:13:06 +00:00
await sails . rm ( sails . config . uploads . prefixForFileDeletion + uploadedSoftware . fd ) ;
2024-12-03 04:28:03 +00:00
sails . log . warn ( ` When attempting to upload a software installer, an unexpected error occurred communicating with the Fleet API, ${ require ( 'util' ) . inspect ( error , { depth : 3 } )} ` ) ;
2024-10-22 02:13:06 +00:00
return { 'softwareUploadFailed' : error } ;
2024-10-15 15:17:05 +00:00
} ) ;
}
// Remove the file from the s3 bucket after it has been sent to the Fleet server.
await sails . rm ( sails . config . uploads . prefixForFileDeletion + uploadedSoftware . fd ) ;
}
return ;
}
} ;