fleet/website/api/controllers/webhooks/receive-usage-analytics.js

module.exports = {


  friendlyName: 'Receive usage analytics',


  description: 'Receive anonymous usage analytics from deployments of Fleet running in production.  (Not fleetctl preview or dev-mode deployments.)',


  inputs: {
    anonymousIdentifier: { required: true, type: 'string', example: '9pnzNmrES3mQG66UQtd29cYTiX2+fZ4CYxDvh495720=', description: 'An anonymous identifier telling us which Fleet deployment this is.', },
    fleetVersion: { required: true, type: 'string', example: 'x.x.x' },
    licenseTier: { type: 'string', isIn: ['free', 'premium', 'unknown'], defaultsTo: 'unknown' },
    numHostsEnrolled: { required: true, type: 'number', min: 0, custom: (num) => Math.floor(num) === num },
    numUsers: { type: 'number', defaultsTo: 0 },
    numTeams: { type: 'number', defaultsTo: 0 },
    numPolicies: { type: 'number', defaultsTo: 0 },
    numLabels: { type: 'number', defaultsTo: 0 },
    softwareInventoryEnabled: { type: 'boolean', defaultsTo: false },
    vulnDetectionEnabled: { type: 'boolean', defaultsTo: false },
    systemUsersEnabled: { type: 'boolean', defaultsTo: false },
    hostStatusWebhookEnabled: { type: 'boolean', defaultsTo: false },
    numWeeklyActiveUsers: { type: 'number', defaultsTo: 0 },
    numWeeklyPolicyViolationDaysActual: { type: 'number', defaultsTo: 0 },
    numWeeklyPolicyViolationDaysPossible: { type: 'number', defaultsTo: 0 },
    hostsEnrolledByOperatingSystem: { type: {}, defaultsTo: {} },
    hostsEnrolledByOrbitVersion: { type: [{orbitVersion: 'string', numHosts: 'number'}], defaultsTo: [] }, // TODO: The name of this parameter does not match naming conventions.
    hostsEnrolledByOsqueryVersion: { type: [{osqueryVersion: 'string', numHosts: 'number'}], defaultsTo: [] }, // TODO: The name of this parameter does not match naming conventions.
    storedErrors: { type: [{}], defaultsTo: [] }, // TODO migrate all rows that have "[]" to {}
    numHostsNotResponding: { type: 'number', defaultsTo: 0, description: 'The number of hosts per deployment that have not submitted results for distibuted queries. A host is counted as not responding if Fleet hasn\'t received a distributed write to requested distibuted queries for the host during the 2-hour interval since the host was last seen. Hosts that have not been seen for 7 days or more are not counted.', },
    organization: { type: 'string', defaultsTo: 'unknown', description: 'For Fleet Premium deployments, the organization registered with the license.', },
  },


  exits: {
    success: { description: 'Analytics data was stored successfully.' },
  },


  fn: async function (inputs) {

    // Create a database record for these usage statistics
    await HistoricalUsageSnapshot.create(inputs);

    if(!sails.config.custom.datadogApiKey) {
      throw new Error('No Datadog API key configured! (Please set sails.config.custom.datadogApiKey)');
    }

    // Store strings and booleans as tags.
    let baseMetricTags = [
      `fleet_version:${inputs.fleetVersion}`,
      `license_tier:${inputs.licenseTier}`,
      `software_inventory_enabled:${inputs.softwareInventoryEnabled}`,
      `vuln_detection_enabled:${inputs.vulnDetectionEnabled}`,
      `system_users_enabled:${inputs.systemUsersEnabled}`,
      `host_status_webhook_enabled:${inputs.hostStatusWebhookEnabled}`,
    ];

    // Create a timestamp in seconds for these metrics
    let metricsTimestampInSeconds = Math.floor(Date.now() / 1000);

    // Build metrics for the usagle statistics that are numbers
    let metricsToSendToDatadog = [
      {
        metric: 'usage_statistics.fleet_server_stats',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: 1
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_hosts_enrolled',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numHostsEnrolled
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_users',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numUsers
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_teams',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numTeams
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_policies',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numPolicies
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_labels',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numLabels
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_weekly_active_users',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numWeeklyActiveUsers
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_weekly_policy_violation_days_actual',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numWeeklyPolicyViolationDaysActual
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_weekly_policy_violation_days_possible',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numWeeklyPolicyViolationDaysPossible
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
      {
        metric: 'usage_statistics.num_hosts_not_responding',
        type: 3,
        points: [{
          timestamp: metricsTimestampInSeconds,
          value: inputs.numHostsNotResponding
        }],
        resources: [{
          name: inputs.anonymousIdentifier,
          type: 'fleet_instance'
        }],
        tags: baseMetricTags,
      },
    ];

    // Build metrics for logged errors
    if(inputs.storedErrors.length > 0) {
      // If inputs.storedErrors is not an empty array, we'll iterate through it to build custom metric for each object in the array
      for(let error of inputs.storedErrors) {
        // Make sure every error object in the storedErrors array has a 'loc' array and a count.
        if((error.loc && _.isArray(error.loc)) && error.count) {
          // Create a new array of tags for this error
          let errorTags = _.clone(baseMetricTags);
          let errorLocation = 1;
          // Create a tag for each error location
          for(let location of error.loc) { // iterate throught the location array of this error
            // Add the error's location as a custom tag (SNAKE_CASED)
            errorTags.push(`error_location_${errorLocation}:${location.replace(/\s/gi, '_')}`);
            errorLocation++;
          }
          let metricToAdd = {
            metric: 'usage_statistics.stored_errors',
            type: 3,
            points: [{timestamp: metricsTimestampInSeconds, value: error.count}],
            resources: [{name: inputs.anonymousIdentifier, type: 'fleet_instance'}],
            tags: errorTags,
          };
          // Add the custom metric to the array of metrics to send to Datadog.
          metricsToSendToDatadog.push(metricToAdd);
        }//fi
      }//∞
    }//fi


    // If inputs.hostsEnrolledByOrbitVersion is not an empty array, we'll iterate through it to build custom metric for each object in the array
    if(inputs.hostsEnrolledByOrbitVersion.length > 0) {
      for(let version of inputs.hostsEnrolledByOrbitVersion) {
        let orbitVersionTags = _.clone(baseMetricTags);
        orbitVersionTags.push(`orbit_version:${version.orbitVersion}`);
        let metricToAdd = {
          metric: 'usage_statistics.host_count_by_orbit_version',
          type: 3,
          points: [{timestamp: metricsTimestampInSeconds, value:version.numHosts}],
          resources: [{name: inputs.anonymousIdentifier, type: 'fleet_instance'}],
          tags: orbitVersionTags,
        };
        // Add the custom metric to the array of metrics to send to Datadog.
        metricsToSendToDatadog.push(metricToAdd);
      }//∞
    }//fi

    // If inputs.hostsEnrolledByOsqueryVersion is not an empty array, we'll iterate through it to build custom metric for each object in the array
    if(inputs.hostsEnrolledByOsqueryVersion.length > 0) {
      for(let version of inputs.hostsEnrolledByOsqueryVersion) {
        let osqueryVersionTags = _.clone(baseMetricTags);
        osqueryVersionTags.push(`osquery_version:${version.osqueryVersion}`);
        let metricToAdd = {
          metric: 'usage_statistics.host_count_by_osquery_version',
          type: 3,
          points: [{timestamp: metricsTimestampInSeconds, value:version.numHosts}],
          resources: [{name: inputs.anonymousIdentifier, type: 'fleet_instance'}],
          tags: osqueryVersionTags,
        };
        // Add the custom metric to the array of metrics to send to Datadog.
        metricsToSendToDatadog.push(metricToAdd);
      }//∞
    }//fi

    // If the hostByOperatingSystem is not an empty object, we'll iterate through the object to build metrics for each type of operating system.
    // See https://fleetdm.com/docs/using-fleet/usage-statistics#what-is-included-in-usage-statistics-in-fleet to see an example of a hostByOperatingSystem send by Fleet instances.
    if(_.keys(inputs.hostsEnrolledByOperatingSystem).length > 0) {
      // Iterate through each array of objects
      for(let operatingSystem in inputs.hostsEnrolledByOperatingSystem) {
        // For every object in the array, we'll send a metric to track host count for each operating system version.
        for(let osVersion of inputs.hostsEnrolledByOperatingSystem[operatingSystem]) {
          // Only continue if the object in the array has a numEnrolled and version value.
          if(osVersion.numEnrolled && osVersion.version) {
            // Clone the baseMetricTags array, each metric will have the operating version name added as a `os_version_name` tag
            let osInfoTags = _.clone(baseMetricTags);
            osInfoTags.push(`os_version_name:${osVersion.version}`);
            let metricToAdd = {
              metric: 'usage_statistics.host_count_by_os_version',
              type: 3,
              points: [{timestamp: metricsTimestampInSeconds, value:osVersion.numEnrolled}],
              resources: [{name: operatingSystem, type: 'os_type'}],
              tags: osInfoTags,
            };
            // Add the custom metric to the array of metrics to send to Datadog.
            metricsToSendToDatadog.push(metricToAdd);
          }//fi
        }//∞
      }//∞
    }//fi

    await sails.helpers.http.post.with({
      url: 'https://api.us5.datadoghq.com/api/v2/series',
      data: {
        series: metricsToSendToDatadog,
      },
      headers: {
        'DD-API-KEY': sails.config.custom.datadogApiKey,
        'Content-Type': 'application/json',
      }
    }).tolerate((err)=>{
      // If there was an error sending metrics to Datadog, we'll log the error in a warning, but we won't throw an error.
      // This way, we'll still return a 200 status to the Fleet instance that sent usage analytics.
      sails.log.warn(`When the receive-usage-analytics webhook tried to send metrics to Datadog, an error occured. Raw error: ${require('util').inspect(err)}`);
    });



  }


};